From 6ba0de42da03187164a045938aad215c38e6347d Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 17 Jun 2021 18:51:10 +0530 Subject: [PATCH 001/123] sysconfig: make _sysconfigdata.py relocatable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: lovetox <8661218+lovetox@users.noreply.github.com> --- Lib/sysconfig.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index acc8d4d1826b8a..7a7c0e20914e5b 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -2,6 +2,7 @@ import os import sys +import textwrap import threading from os.path import realpath @@ -524,11 +525,30 @@ def _generate_posix_vars(): os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') + replacement = """ + keys_to_replace = [ + 'BINDIR', 'BINLIBDEST', 'CONFINCLUDEDIR', + 'CONFINCLUDEPY', 'DESTDIRS', 'DESTLIB', 'DESTSHARED', + 'INCLDIRSTOMAKE', 'INCLUDEDIR', 'INCLUDEPY', + 'LIBDEST', 'LIBDIR', 'LIBPC', 'LIBPL', 'MACHDESTLIB', + 'MANDIR', 'SCRIPTDIR', 'datarootdir', 'exec_prefix', + 'TZPATH', + ] + + prefix = build_time_vars['BINDIR'][:-4] + + for key in keys_to_replace: + value = build_time_vars[key] + build_time_vars[key] = value.replace(prefix, sys.base_prefix) + """ + with open(destfile, 'w', encoding='utf8') as f: + f.write('import sys\n') f.write('# system configuration generated and used by' ' the sysconfig module\n') f.write('build_time_vars = ') pprint.pprint(vars, stream=f) + f.write('\n%s' % textwrap.dedent(replacement)) # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: From 38561b675915568dd7521452d59793b0ebcd2f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:12 +0530 Subject: [PATCH 002/123] build: add --with-nt-threads and make it default on mingw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Naveen M K Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Include/internal/pycore_condvar.h | 10 ++ Include/pythread.h | 6 ++ Modules/_multiprocessing/multiprocessing.h | 5 +- configure.ac | 102 ++++++++++++++++++++- pyconfig.h.in | 3 + 5 files changed, 122 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 2177bed31fcd27..c46486f0947b8d 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -5,6 +5,12 @@ # error "this header requires Py_BUILD_CORE define" #endif +#ifdef __MINGW32__ +# if !defined(HAVE_PTHREAD_H) || defined(NT_THREADS) +# undef _POSIX_THREADS +# endif +#endif + #ifndef _POSIX_THREADS /* This means pthreads are not implemented in libc headers, hence the macro not present in unistd.h. But they still can be implemented as an external @@ -41,6 +47,10 @@ # define WIN32_LEAN_AND_MEAN #endif #include // CRITICAL_SECTION +/* winpthreads are involved via windows header, so need undef _POSIX_THREADS after header include */ +#if defined(_POSIX_THREADS) +#undef _POSIX_THREADS +#endif /* options */ /* non-emulated condition variables are provided for those that want diff --git a/Include/pythread.h b/Include/pythread.h index 63714437c496b7..5e84e3cbefc330 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -7,6 +7,12 @@ typedef void *PyThread_type_lock; extern "C" { #endif +#ifdef __MINGW32__ +# if !defined(HAVE_PTHREAD_H) || defined(NT_THREADS) +# undef _POSIX_THREADS +# endif +#endif + /* Return status codes for Python lock acquisition. Chosen for maximum * backwards compatibility, ie failure -> 0, success -> 1. */ typedef enum PyLockStatus { diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index dfc2a8e0799a60..dc92a233d374c6 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -23,7 +23,10 @@ # endif # define SEM_HANDLE HANDLE # define SEM_VALUE_MAX LONG_MAX -# define HAVE_MP_SEMAPHORE +# define HAVE_MP_SEMAPHORE +# if defined(HAVE_SEM_OPEN) && defined(_POSIX_THREADS) +# include +# endif #else # include /* O_CREAT and O_EXCL */ # if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) diff --git a/configure.ac b/configure.ac index 1a02d19f1b2a3f..de008a9be75b0e 100644 --- a/configure.ac +++ b/configure.ac @@ -2721,6 +2721,53 @@ then BASECFLAGS="$BASECFLAGS $ac_arch_flags" fi +dnl NOTE: +dnl - GCC 4.4+ for mingw* require and use posix threads(pthreads-w32) +dnl - Host may contain installed pthreads-w32. +dnl - On windows platform only NT-thread model is supported. +dnl To avoid miss detection scipt first will check for NT-thread model +dnl and if is not found will try to detect build options for pthread +dnl model. Autodetection could be overiden if variable with_nt_threads +dnl is set in "Site Configuration" (see autoconf manual). +dnl If NT-thread model is enabled script skips some checks that +dnl impact build process. When a new functionality is added, developers +dnl are responsible to update configure script to avoid thread models +dnl to be mixed. + +AC_MSG_CHECKING([for --with-nt-threads]) +AC_ARG_WITH(nt-threads, + AS_HELP_STRING([--with-nt-threads], [build with windows threads (default is system-dependent)]), +[ + case $withval in + no) with_nt_threads=no;; + yes) with_nt_threads=yes;; + *) with_nt_threads=yes;; + esac +], [ + case $host in + *-*-mingw*) with_nt_threads=yes;; + *) with_nt_threads=no;; + esac +]) +AC_MSG_RESULT([$with_nt_threads]) + +if test $with_nt_threads = yes ; then +AC_MSG_CHECKING([whether linking with nt-threads work]) +AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[]],[[_beginthread(0, 0, 0);]]) + ], + [AC_MSG_RESULT([yes])], + [AC_MSG_ERROR([failed to link with nt-threads])]) +fi + +if test $with_nt_threads = yes ; then + dnl temporary default flag to avoid additional pthread checks + dnl and initilize other ac..thread flags to no + ac_cv_pthread_is_default=no + ac_cv_kthread=no + ac_cv_pthread=no + dnl ac_cv_kpthread is set to no if default is yes (see below) +else # On some compilers, pthreads are available without further options # (e.g. MacOS X). On some of these systems, the compiler will not # complain if unaccepted options are passed (e.g. gcc on Mac OS X). @@ -2832,6 +2879,8 @@ int main(void){ CC="$ac_save_cc"]) fi +fi + # If we have set a CC compiler flag for thread support then # check if it works for CXX, too. if test ! -z "$CXX" @@ -2851,6 +2900,10 @@ elif test "$ac_cv_pthread" = "yes" then CXX="$CXX -pthread" ac_cv_cxx_thread=yes +elif test "$with_nt_threads" = "yes" +then + dnl set to always to skip extra pthread check below + ac_cv_cxx_thread=always else ac_cv_cxx_thread=no fi @@ -2892,8 +2945,8 @@ AC_CHECK_HEADERS([ \ alloca.h asm/types.h bluetooth.h conio.h crypt.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ ieeefp.h io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ linux/random.h linux/soundcard.h \ - linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ - sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ + linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pty.h \ + setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/pidfd.h sys/poll.h \ sys/random.h sys/resource.h sys/select.h sys/sendfile.h sys/socket.h sys/soundcard.h sys/stat.h \ @@ -2904,6 +2957,14 @@ AC_CHECK_HEADERS([ \ AC_HEADER_DIRENT AC_HEADER_MAJOR +# If using nt threads, don't look for pthread.h or thread.h +if test "x$with_nt_threads" = xno ; then +AC_HEADER_STDC +AC_CHECK_HEADERS(pthread.h sched.h thread.h) +AC_HEADER_DIRENT +AC_HEADER_MAJOR +fi + # bluetooth/bluetooth.h has been known to not compile with -std=c99. # http://permalink.gmane.org/gmane.linux.bluez.kernel/22294 SAVE_CFLAGS=$CFLAGS @@ -3116,6 +3177,10 @@ elif test "$ac_cv_pthread" = "yes" then CC="$CC -pthread" fi +if test $with_nt_threads = yes ; then + dnl skip check for pthread_t if NT-thread model is enabled + ac_cv_have_pthread_t=skip +else AC_CACHE_CHECK([for pthread_t], [ac_cv_have_pthread_t], [ AC_COMPILE_IFELSE([ AC_LANG_PROGRAM([[@%:@include ]], [[pthread_t x; x = *(pthread_t*)0;]]) @@ -3147,7 +3212,7 @@ AS_VAR_IF([ac_cv_pthread_key_t_is_arithmetic_type], [yes], [ AC_DEFINE([PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT], [1], [Define if pthread_key_t is compatible with int.]) ]) - +fi CC="$ac_save_cc" AC_MSG_CHECKING([for --enable-framework]) @@ -3679,10 +3744,15 @@ AS_VAR_IF([have_uuid], [missing], [ AS_VAR_IF([have_uuid], [missing], [have_uuid=no]) +if test $with_nt_threads = yes ; then + dnl do not search for sem_init if NT-thread model is enabled + : +else # 'Real Time' functions on Solaris # posix4 on Solaris 2.6 # pthread (first!) on Linux AC_SEARCH_LIBS([sem_init], [pthread rt posix4]) +fi # check if we need libintl for locale functions AC_CHECK_LIB([intl], [textdomain], @@ -4402,6 +4472,11 @@ then CXX="$CXX -pthread" fi posix_threads=yes +elif test $with_nt_threads = yes +then + posix_threads=no + AC_DEFINE(NT_THREADS, 1, + [Define to 1 if you want to use native NT threads]) else if test ! -z "$withval" -a -d "$withval" then LDFLAGS="$LDFLAGS -L$withval" @@ -4918,6 +4993,15 @@ else fi # checks for library functions +if test $with_nt_threads = yes ; then + dnl GCC(mingw) 4.4+ require and use posix threads(pthreads-w32) + dnl and host may contain installed pthreads-w32. + dnl Skip checks for some functions declared in pthreads-w32 if + dnl NT-thread model is enabled. + ac_cv_func_pthread_kill=skip + ac_cv_func_sem_open=skip + ac_cv_func_sched_setscheduler=skip +fi AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock close_range confstr \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ @@ -5814,6 +5898,10 @@ dnl actually works. For FreeBSD versions <= 7.2, dnl the kernel module that provides POSIX semaphores dnl isn't loaded by default, so an attempt to call dnl sem_open results in a 'Signal 12' error. +if test $with_nt_threads = yes ; then + dnl skip posix semaphores test if NT-thread model is enabled + ac_cv_posix_semaphores_enabled=no +fi AC_CACHE_CHECK([whether POSIX semaphores are enabled], [ac_cv_posix_semaphores_enabled], AC_RUN_IFELSE([ AC_LANG_SOURCE([ @@ -5847,6 +5935,14 @@ AS_VAR_IF([ac_cv_posix_semaphores_enabled], [no], [ ]) dnl Multiprocessing check for broken sem_getvalue +if test $with_nt_threads = yes ; then + dnl Skip test if NT-thread model is enabled. + dnl NOTE the test case below fail for pthreads-w32 as: + dnl - SEM_FAILED is not defined; + dnl - sem_open is a stub; + dnl - sem_getvalue work(!). + ac_cv_broken_sem_getvalue=skip +fi AC_CACHE_CHECK([for broken sem_getvalue], [ac_cv_broken_sem_getvalue], AC_RUN_IFELSE([ AC_LANG_SOURCE([ diff --git a/pyconfig.h.in b/pyconfig.h.in index df4d29fe5497a4..b359e13b5d9836 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1552,6 +1552,9 @@ /* Define if mvwdelch in curses.h is an expression. */ #undef MVWDELCH_IS_EXPRESSION +/* Define to 1 if you want to use native NT threads */ +#undef NT_THREADS + /* Define to the address where bug reports for this package should be sent. */ #undef PACKAGE_BUGREPORT From d565b952e650208537b188f2cd49c902dac05f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:13 +0530 Subject: [PATCH 003/123] Define `MS_WINDOWS` (and others) when compiling with MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is used throughout the Python code but is not defined when compiling with Mingw-w64 compilers, so define it. In MSVC these are defined in PC/pyconfig.h, we have to define it manually because pyconfig.h is created by autotools. Co-authored-by: Алексей Co-authored-by: Christoph Reiter Co-authored-by: Naveen M K --- Include/pyport.h | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Include/pyport.h b/Include/pyport.h index e2bac3bf504261..3d70b8e07815b1 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -53,6 +53,44 @@ #endif +#ifdef __MINGW32__ +/* Translate GCC[mingw*] platform specific defines to those + * used in python code. + */ +#if !defined(MS_WIN64) && defined(_WIN64) +# define MS_WIN64 +#endif +#if !defined(MS_WIN32) && defined(_WIN32) +# define MS_WIN32 +#endif +#if !defined(MS_WINDOWS) && defined(MS_WIN32) +# define MS_WINDOWS +#endif + +#if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN) || defined(Py_BUILD_CORE_MODULE) +#include + +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#define MS_WINDOWS_DESKTOP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +#define MS_WINDOWS_APP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_SYSTEM) +#define MS_WINDOWS_SYSTEM +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_GAMES) +#define MS_WINDOWS_GAMES +#endif + +/* Define to 1 if you support windows console io */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) +#define HAVE_WINDOWS_CONSOLE_IO 1 +#endif +#endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ + +#endif /* __MINGW32__*/ + /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. From 396c8847eda624428aeb7414f15bb5f5f864d9c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:14 +0530 Subject: [PATCH 004/123] configure: add `MACHDEP` and `platform` on MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- configure.ac | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/configure.ac b/configure.ac index de008a9be75b0e..d003a531890741 100644 --- a/configure.ac +++ b/configure.ac @@ -550,6 +550,9 @@ then *-*-cygwin*) ac_sys_system=Cygwin ;; + *-*-mingw*) + ac_sys_system=MINGW + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -585,6 +588,7 @@ then linux*) MACHDEP="linux";; cygwin*) MACHDEP="cygwin";; darwin*) MACHDEP="darwin";; + mingw*) MACHDEP="win32";; '') MACHDEP="unknown";; esac @@ -618,6 +622,9 @@ if test "$cross_compiling" = yes; then ;; wasm32-*-* | wasm64-*-*) _host_cpu=$host_cpu + ;; + *-*-mingw*) + _host_cpu= ;; *) # for now, limit cross builds to known configurations @@ -625,6 +632,14 @@ if test "$cross_compiling" = yes; then AC_MSG_ERROR([cross build not supported for $host]) esac _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + + case "$host_os" in + mingw*) + # As sys.platform() return 'win32' to build python and extantions + # we will use 'mingw' (in setup.py and etc.) + _PYTHON_HOST_PLATFORM=mingw + ;; + esac fi # Some systems cannot stand _XOPEN_SOURCE being defined at all; they From 989e86b3b48e8fed1a0053eed7ab93ba62e33931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:15 +0530 Subject: [PATCH 005/123] Add default configuration for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Misc/config_mingw | 12 ++++++++++++ Misc/cross_mingw32 | 11 +++++++++++ configure.ac | 22 ++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 Misc/config_mingw create mode 100644 Misc/cross_mingw32 diff --git a/Misc/config_mingw b/Misc/config_mingw new file mode 100644 index 00000000000000..513065dec9b717 --- /dev/null +++ b/Misc/config_mingw @@ -0,0 +1,12 @@ +# configure defaults for mingw* hosts + +# mingw functions to ignore +ac_cv_func_ftruncate=ignore # implement it as _chsize + +# mingw-w64 functions to ignore +ac_cv_func_truncate=ignore +ac_cv_func_alarm=ignore + +# files to ignore +ac_cv_file__dev_ptmx=ignore #NOTE: under MSYS environment device exist +ac_cv_file__dev_ptc=no diff --git a/Misc/cross_mingw32 b/Misc/cross_mingw32 new file mode 100644 index 00000000000000..03fde9e6af3ad7 --- /dev/null +++ b/Misc/cross_mingw32 @@ -0,0 +1,11 @@ +# configure defaults for mingw32 host if cross-build + +ac_cv_little_endian_double=yes +ac_cv_big_endian_double=no +ac_cv_mixed_endian_double=no + +ac_cv_tanh_preserves_zero_sign=yes + +ac_cv_wchar_t_signed=no + +ac_cv_have_size_t_format=no diff --git a/configure.ac b/configure.ac index d003a531890741..8b11800daf52c4 100644 --- a/configure.ac +++ b/configure.ac @@ -1205,6 +1205,28 @@ AC_CACHE_CHECK([for -Wl,--no-as-needed], [ac_cv_wl_no_as_needed], [ ]) AC_SUBST([NO_AS_NEEDED]) +# initialize default configuration +py_config= +case $host in + *-*-mingw*) py_config=mingw ;; +esac +if test -n "$py_config" ; then + AC_MSG_NOTICE([loading configure defaults from .../Misc/config_$py_config"]) + . "$srcdir/Misc/config_$py_config" +fi + +# initialize defaults for cross-builds +if test "$cross_compiling" = yes; then + py_config=$host_os + case $py_config in + mingw32*) py_config=mingw32 ;; + esac + if test -f "$srcdir/Misc/cross_$py_config" ; then + AC_MSG_NOTICE([loading cross defaults from .../Misc/cross_$py_config"]) + . "$srcdir/Misc/cross_$py_config" + fi +fi + AC_MSG_CHECKING([for the Android API level]) cat > conftest.c < Date: Thu, 17 Jun 2021 18:51:16 +0530 Subject: [PATCH 006/123] configure: enable largefile support by default for Mingw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Largefile is supported on Windows. Co-authored-by: Naveen M K Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- configure.ac | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure.ac b/configure.ac index 8b11800daf52c4..6914b148382a72 100644 --- a/configure.ac +++ b/configure.ac @@ -3184,6 +3184,10 @@ dnl LFS does not work with Emscripten 3.1 AS_CASE([$ac_sys_system], [Emscripten], [have_largefile_support="no"] ) +dnl Activate on windows platforms (32&64-bit) where off_t(4) < fpos_t(8) +AS_CASE([$ac_sys_system], + [MINGW], [have_largefile_support="yes"] +) AS_VAR_IF([have_largefile_support], [yes], [ AC_DEFINE([HAVE_LARGEFILE_SUPPORT], [1], [Defined to enable large file support when an off_t is bigger than a long From 5f97c1f9acd908c10f0bcd90ff6bdd22888bf2bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:18 +0530 Subject: [PATCH 007/123] Add `PC/` to `CPPFLAGS` and to `SRCDIRS` on Mingw Co-authored-by: Naveen M K --- configure.ac | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/configure.ac b/configure.ac index 6914b148382a72..aeb992878c22be 100644 --- a/configure.ac +++ b/configure.ac @@ -6913,6 +6913,16 @@ do THREADHEADERS="$THREADHEADERS \$(srcdir)/$h" done +case $host in + *-*-mingw*) + dnl Required for windows builds as Objects/exceptions.c require + dnl "errmap.h" from $srcdir/PC. + dnl Note we cannot use BASECPPFLAGS as autogenerated pyconfig.h + dnl has to be before customized located in ../PC. + CPPFLAGS="-I\$(srcdir)/PC $CPPFLAGS" + ;; +esac + AC_SUBST([SRCDIRS]) SRCDIRS="\ Modules \ @@ -6935,6 +6945,11 @@ SRCDIRS="\ Python \ Python/frozen_modules \ Python/deepfreeze" + +case $host in + *-*-mingw*) SRCDIRS="$SRCDIRS PC";; +esac + AC_MSG_CHECKING([for build directories]) for dir in $SRCDIRS; do if test ! -d $dir; then From cda6ac76637f6df19fd9ab023f3c64c3f31e7d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:19 +0530 Subject: [PATCH 008/123] posixmodule.c: Define necessary headers for compiling on MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also, build `nt` module instead of `posix` when compiling with MINGW. Co-authored-by: Naveen M K Co-authored-by: Алексей --- Modules/Setup.bootstrap.in | 2 +- Modules/posixmodule.c | 28 ++++++++++++++++++++++++---- configure.ac | 8 ++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 8ef0f203a82a8e..1f21d3578ae358 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -8,7 +8,7 @@ # module C APIs are used in core atexit atexitmodule.c faulthandler faulthandler.c -posix posixmodule.c +@INITSYS@ posixmodule.c _signal signalmodule.c _tracemalloc _tracemalloc.c diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c21c6f06c729de..1a62ddd91d3b5e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -369,6 +369,26 @@ corresponding Unix manual entries for more information on calls."); # define HAVE_PIPE 1 # define HAVE_FSYNC 1 # define fsync _commit +#elif defined(__MINGW32__) /* GCC for windows hosts */ +/* getlogin is detected by configure on mingw-w64 */ +# undef HAVE_GETLOGIN +/*# define HAVE_GETCWD 1 - detected by configure*/ +# define HAVE_GETPPID 1 +# define HAVE_GETLOGIN 1 +# define HAVE_SPAWNV 1 +# define HAVE_WSPAWNV 1 +# define HAVE_WEXECV 1 +/*# define HAVE_EXECV 1 - detected by configure*/ +# define HAVE_PIPE 1 +# define HAVE_POPEN 1 +# define HAVE_SYSTEM 1 +# define HAVE_CWAIT 1 +# define HAVE_FSYNC 1 +# define fsync _commit +# include +# ifndef _MAX_ENV +# define _MAX_ENV 32767 +# endif #endif /* ! __WATCOMC__ || __QNX__ */ /*[clinic input] @@ -446,7 +466,7 @@ extern char *ctermid_r(char *); # endif #endif -#ifdef _MSC_VER +#ifdef MS_WINDOWS # ifdef HAVE_DIRECT_H # include # endif @@ -457,7 +477,7 @@ extern char *ctermid_r(char *); # include # endif # include -#endif /* _MSC_VER */ +#endif /* MS_WINDOWS */ #ifndef MAXPATHLEN # if defined(PATH_MAX) && PATH_MAX > 1024 @@ -1594,9 +1614,9 @@ _Py_Sigset_Converter(PyObject *obj, void *addr) ** man environ(7). */ #include -#elif !defined(_MSC_VER) && (!defined(__WATCOMC__) || defined(__QNX__) || defined(__VXWORKS__)) +#elif !defined(MS_WINDOWS) && (!defined(__WATCOMC__) || defined(__QNX__) || defined(__VXWORKS__)) extern char **environ; -#endif /* !_MSC_VER */ +#endif /* !MS_WINDOWS */ static PyObject * convertenviron(void) diff --git a/configure.ac b/configure.ac index aeb992878c22be..1bc49743c3d25a 100644 --- a/configure.ac +++ b/configure.ac @@ -755,6 +755,14 @@ then [Define to include mbstate_t for mbrtowc]) fi +AC_MSG_CHECKING([for init system calls]) +AC_SUBST(INITSYS) +case $host in + *-*-mingw*) INITSYS=nt;; + *) INITSYS=posix;; +esac +AC_MSG_RESULT([$INITSYS]) + # Record the configure-time value of MACOSX_DEPLOYMENT_TARGET, # it may influence the way we can build extensions, so distutils # needs to check it From 8046053fef0ce72a6b771258de76bd5d651864cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:20 +0530 Subject: [PATCH 009/123] Build winreg by default when compiling for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter Co-authored-by: Naveen M K --- Modules/Setup.bootstrap.in | 3 +++ PC/winreg.c | 19 +++++++++++++++++++ configure.ac | 7 +++++++ 3 files changed, 29 insertions(+) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 1f21d3578ae358..3def7e8f810731 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -34,3 +34,6 @@ _symtable symtablemodule.c # for systems without $HOME env, used by site._getuserbase() @MODULE_PWD_TRUE@pwd pwdmodule.c + +# build-in modules for windows platform: +@USE_WIN32_MODULE@winreg ../PC/winreg.c diff --git a/PC/winreg.c b/PC/winreg.c index e2d5322f458c2a..368291d9dafa4e 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -19,6 +19,25 @@ #include "structmember.h" // PyMemberDef #include +#ifndef SIZEOF_HKEY +/* used only here */ +#if defined(MS_WIN64) +# define SIZEOF_HKEY 8 +#elif defined(MS_WIN32) +# define SIZEOF_HKEY 4 +#else +# error "SIZEOF_HKEY is not defined" +#endif +#endif + +#ifndef REG_LEGAL_CHANGE_FILTER +#define REG_LEGAL_CHANGE_FILTER (\ + REG_NOTIFY_CHANGE_NAME |\ + REG_NOTIFY_CHANGE_ATTRIBUTES |\ + REG_NOTIFY_CHANGE_LAST_SET |\ + REG_NOTIFY_CHANGE_SECURITY ) +#endif + #if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) typedef struct { diff --git a/configure.ac b/configure.ac index 1bc49743c3d25a..c2b1e6cb5c7323 100644 --- a/configure.ac +++ b/configure.ac @@ -4487,6 +4487,13 @@ done IFS=$as_save_IFS AC_MSG_RESULT([$DBM_CFLAGS $DBM_LIBS]) +# Determine if windows modules should be used. +AC_SUBST(USE_WIN32_MODULE) +USE_WIN32_MODULE='#' +case $host in + *-*-mingw*) USE_WIN32_MODULE=;; +esac + # Templates for things AC_DEFINEd more than once. # For a single AC_DEFINE, no template is needed. AH_TEMPLATE([_REENTRANT], From 65a9539572746bdf4c5595e81ee20521521e61f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:23 +0530 Subject: [PATCH 010/123] pyport.h: Add support for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/pyport.h | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index 3d70b8e07815b1..ddff496ba80544 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -532,12 +532,12 @@ extern char * _getpty(int *, int, mode_t, int); */ /* - All windows ports, except cygwin, are handled in PC/pyconfig.h. + Only MSVC windows ports is handled in PC/pyconfig.h. - Cygwin is the only other autoconf platform requiring special + Cygwin and Mingw is the only other autoconf platform requiring special linkage handling and it uses __declspec(). */ -#if defined(__CYGWIN__) +#if defined(__CYGWIN__) || defined(__MINGW32__) # define HAVE_DECLSPEC_DLL #endif @@ -550,21 +550,23 @@ extern char * _getpty(int *, int, mode_t, int); # define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE # define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE /* module init functions inside the core need no external linkage */ - /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) + /* except for Cygwin/Mingw to handle embedding */ +# if defined(__CYGWIN__) || defined(__MINGW32__) # define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# else /* __CYGWIN__ */ +# else /* __CYGWIN__ || __MINGW32__*/ # define PyMODINIT_FUNC PyObject* -# endif /* __CYGWIN__ */ +# endif /* __CYGWIN__ || __MINGW32__*/ # else /* Py_BUILD_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ /* failures similar to those described at the bottom of 4.1: */ /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ -# if !defined(__CYGWIN__) +# if defined(__CYGWIN__) || defined(__MINGW32__) +# define PyAPI_FUNC(RTYPE) RTYPE +# else # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE -# endif /* !__CYGWIN__ */ +# endif /* __CYGWIN__ || __MINGW32__*/ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE /* module init functions outside the core must be exported */ # if defined(__cplusplus) From 17ac42a9e03997001248860bbfda206e12198ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:24 +0530 Subject: [PATCH 011/123] configure: add options so that shared build is possible on MINGW Co-authored-by: Naveen M K --- configure.ac | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c2b1e6cb5c7323..d4e8b5f27558a0 100644 --- a/configure.ac +++ b/configure.ac @@ -1552,6 +1552,11 @@ if test $enable_shared = "yes"; then LDLIBRARY='libpython$(LDVERSION).dll.a' DLLLIBRARY='libpython$(LDVERSION).dll' ;; + MINGW*) + LDLIBRARY='libpython$(LDVERSION).dll.a' + DLLLIBRARY='libpython$(LDVERSION).dll' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + ;; SunOS*) LDLIBRARY='libpython$(LDVERSION).so' BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' @@ -1602,6 +1607,9 @@ else # shared is disabled BLDLIBRARY='$(LIBRARY)' LDLIBRARY='libpython$(LDVERSION).dll.a' ;; + MINGW*) + LDLIBRARY='libpython$(LDVERSION).a' + ;; esac fi @@ -3545,6 +3553,10 @@ then CYGWIN*) LDSHARED="gcc -shared -Wl,--enable-auto-image-base" LDCXXSHARED="g++ -shared -Wl,--enable-auto-image-base";; + MINGW*) + LDSHARED='$(CC) -shared -Wl,--enable-auto-image-base' + LDCXXSHARED='$(CXX) -shared -Wl,--enable-auto-image-base' + ;; *) LDSHARED="ld";; esac fi @@ -6934,7 +6946,8 @@ case $host in dnl "errmap.h" from $srcdir/PC. dnl Note we cannot use BASECPPFLAGS as autogenerated pyconfig.h dnl has to be before customized located in ../PC. - CPPFLAGS="-I\$(srcdir)/PC $CPPFLAGS" + dnl (-I. at end is workaround for setup.py logic) + CPPFLAGS="-I\$(srcdir)/PC $CPPFLAGS -I." ;; esac From 3f6018afd94e038f1aa28f92431f4163bd4b53ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:25 +0530 Subject: [PATCH 012/123] Add dynamic loading support for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Makefile.pre.in | 6 ++++++ Python/dynload_win.c | 9 +++++++-- configure.ac | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 7ca3dc62c01555..2d8145ff16508e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1378,6 +1378,12 @@ Python/dynload_hpux.o: $(srcdir)/Python/dynload_hpux.c Makefile -DSHLIB_EXT='"$(EXT_SUFFIX)"' \ -o $@ $(srcdir)/Python/dynload_hpux.c +Python/dynload_win.o: $(srcdir)/Python/dynload_win.c Makefile + $(CC) -c $(PY_CORE_CFLAGS) \ + -DSHLIB_SUFFIX='"$(SHLIB_SUFFIX)"' \ + -DEXT_SUFFIX='"$(EXT_SUFFIX)"' \ + -o $@ $(srcdir)/Python/dynload_win.c + Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h $(CC) -c $(PY_CORE_CFLAGS) \ -DABIFLAGS='"$(ABIFLAGS)"' \ diff --git a/Python/dynload_win.c b/Python/dynload_win.c index acab05e2c6def3..4bb90ba198b887 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -29,6 +29,12 @@ #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" const char *_PyImport_DynLoadFiletab[] = { +#ifdef EXT_SUFFIX + EXT_SUFFIX, /* include SOABI flags where is encoded debug */ +#endif +#ifdef SHLIB_SUFFIX + "-abi" PYTHON_ABI_STRING SHLIB_SUFFIX, +#endif PYD_TAGGED_SUFFIX, PYD_UNTAGGED_SUFFIX, NULL @@ -251,8 +257,7 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, ensure DLLs adjacent to the PYD are preferred. */ Py_BEGIN_ALLOW_THREADS hDLL = LoadLibraryExW(wpathname, NULL, - LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | - LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR); + LOAD_WITH_ALTERED_SEARCH_PATH); Py_END_ALLOW_THREADS PyMem_Free(wpathname); diff --git a/configure.ac b/configure.ac index d4e8b5f27558a0..1863fa8ed4f270 100644 --- a/configure.ac +++ b/configure.ac @@ -3424,6 +3424,9 @@ if test -z "$SHLIB_SUFFIX"; then CYGWIN*) SHLIB_SUFFIX=.dll;; *) SHLIB_SUFFIX=.so;; esac + case $host_os in + mingw*) SHLIB_SUFFIX=.dll;; + esac fi AC_MSG_RESULT([$SHLIB_SUFFIX]) @@ -5036,6 +5039,13 @@ then fi ;; esac + case $host in + *-*-mingw*) + DYNLOADFILE="dynload_win.o" + extra_machdep_objs="$extra_machdep_objs PC/dl_nt.o" + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"'" + ;; + esac fi AC_MSG_RESULT([$DYNLOADFILE]) if test "$DYNLOADFILE" != "dynload_stub.o" @@ -6925,6 +6935,12 @@ case "$ac_cv_computed_gotos" in yes*) AC_DEFINE([HAVE_COMPUTED_GOTOS], [1], [Define if the C compiler supports computed gotos.]) esac +case $host_os in + mingw*) + dnl Synchronized with _PyImport_DynLoadFiletab (dynload_win.c) + dnl Do not use more then one dot on this platform ! + EXT_SUFFIX=-$SOABI$SHLIB_SUFFIX;; +esac case $ac_sys_system in AIX*) From 1dd0573aac766ba4464b5afb2bb7be86f955ad60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:26 +0530 Subject: [PATCH 013/123] Ignore main program for frozen scripts on MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Makefile.pre.in | 2 +- configure.ac | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2d8145ff16508e..5e005b63e94597 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -383,7 +383,7 @@ PYTHON_OBJS= \ Python/errors.o \ Python/flowgraph.o \ Python/frame.o \ - Python/frozenmain.o \ + @PYTHON_OBJS_FROZENMAIN@ \ Python/future.o \ Python/getargs.o \ Python/getcompiler.o \ diff --git a/configure.ac b/configure.ac index 1863fa8ed4f270..e03f864b93b120 100644 --- a/configure.ac +++ b/configure.ac @@ -6967,6 +6967,16 @@ case $host in ;; esac +dnl Python interpreter main program for frozen scripts +AC_SUBST(PYTHON_OBJS_FROZENMAIN) +PYTHON_OBJS_FROZENMAIN="Python/frozenmain.o" +case $host in + *-*-mingw*) + dnl 'PC/frozen_dllmain.c' - not yet + PYTHON_OBJS_FROZENMAIN= + ;; +esac + AC_SUBST([SRCDIRS]) SRCDIRS="\ Modules \ From 9c79fca7a73c2161df385a8580984a57a9a76f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:30 +0530 Subject: [PATCH 014/123] Detect winsock2 and setup _socket module on MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Misc/config_mingw | 3 +++ Modules/socketmodule.c | 6 ++++++ configure.ac | 38 ++++++++++++++++++++++++++++++++------ pyconfig.h.in | 4 ++-- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/Misc/config_mingw b/Misc/config_mingw index 513065dec9b717..9be43fd3a11407 100644 --- a/Misc/config_mingw +++ b/Misc/config_mingw @@ -10,3 +10,6 @@ ac_cv_func_alarm=ignore # files to ignore ac_cv_file__dev_ptmx=ignore #NOTE: under MSYS environment device exist ac_cv_file__dev_ptc=no + +# force detection of winsock2 functionality - require wxp or newer +ac_cv_func_getpeername=yes diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 27afd73d9704db..7038e10d663e41 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -282,6 +282,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include /* Macros based on the IPPROTO enum, see: https://bugs.python.org/issue29515 */ +#ifdef _MSC_VER #define IPPROTO_ICMP IPPROTO_ICMP #define IPPROTO_IGMP IPPROTO_IGMP #define IPPROTO_GGP IPPROTO_GGP @@ -312,6 +313,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #define IPPROTO_PGM IPPROTO_PGM // WinSock2 only #define IPPROTO_L2TP IPPROTO_L2TP // WinSock2 only #define IPPROTO_SCTP IPPROTO_SCTP // WinSock2 only +#endif /* _MSC_VER */ /* Provides the IsWindows7SP1OrGreater() function */ #include @@ -426,6 +428,10 @@ remove_unusable_flags(PyObject *m) /* Do not include addrinfo.h for MSVC7 or greater. 'addrinfo' and * EAI_* constants are defined in (the already included) ws2tcpip.h. */ +#elif defined(__MINGW32__) + /* Do not include addrinfo.h as minimum supported version is + * _WIN32_WINNT >= WindowsXP(0x0501) + */ #else # include "addrinfo.h" #endif diff --git a/configure.ac b/configure.ac index e03f864b93b120..37554f7facf8ea 100644 --- a/configure.ac +++ b/configure.ac @@ -4188,6 +4188,12 @@ AS_CASE([$ac_sys_system], [OSSAUDIODEV_LIBS=""] ) +dnl On MINGW, you need to link against ws2_32 and iphlpapi for sockets to work +AS_CASE([$MACHDEP], + [win32], [SOCKET_LIBS="-lws2_32 -liphlpapi -lrpcrt4"], + [SOCKET_LIBS=""] +) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -5651,18 +5657,33 @@ if test $ac_cv_header_time_altzone = yes; then [Define this if your time.h defines altzone.]) fi +AC_CHECK_HEADERS([ws2tcpip.h]) AC_CACHE_CHECK([for addrinfo], [ac_cv_struct_addrinfo], -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], [[struct addrinfo a]])], +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +#ifdef HAVE_WS2TCPIP_H +# include +#else +# include +#endif]], + [[struct addrinfo a]])], [ac_cv_struct_addrinfo=yes], [ac_cv_struct_addrinfo=no])) if test $ac_cv_struct_addrinfo = yes; then - AC_DEFINE([HAVE_ADDRINFO], [1], [struct addrinfo (netdb.h)]) + AC_DEFINE([HAVE_ADDRINFO], [1], [struct addrinfo]) fi AC_CACHE_CHECK([for sockaddr_storage], [ac_cv_struct_sockaddr_storage], AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ -# include -@%:@ include ]], [[struct sockaddr_storage s]])], +#ifdef HAVE_WS2TCPIP_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif]], + [[struct sockaddr_storage s]])], [ac_cv_struct_sockaddr_storage=yes], [ac_cv_struct_sockaddr_storage=no])) if test $ac_cv_struct_sockaddr_storage = yes; then @@ -6855,8 +6876,11 @@ AC_CHECK_TYPE( [socklen_t], [], [AC_DEFINE( [socklen_t], [int], - [Define to `int' if does not define.] + [Define to `int' if or does not define.] )], [ +#ifdef HAVE_WS2TCPIP_H +#include +#endif #ifdef HAVE_SYS_TYPES_H #include #endif @@ -7665,7 +7689,9 @@ PY_STDLIB_MOD([mmap], PY_STDLIB_MOD([_socket], [], m4_flatten([test "$ac_cv_header_sys_socket_h" = "yes" -a "$ac_cv_header_sys_types_h" = "yes" - -a "$ac_cv_header_netinet_in_h" = "yes"])) + -a "$ac_cv_header_netinet_in_h" = "yes" + -o "$MACHDEP" = "win32"]), + [], [$SOCKET_LIBS]) dnl platform specific extensions PY_STDLIB_MOD([grp], [], [test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes]) diff --git a/pyconfig.h.in b/pyconfig.h.in index b359e13b5d9836..398b64f2f0be49 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -59,7 +59,7 @@ /* Define to 1 if you have the `acosh' function. */ #undef HAVE_ACOSH -/* struct addrinfo (netdb.h) */ +/* struct addrinfo */ #undef HAVE_ADDRINFO /* Define to 1 if you have the `alarm' function. */ @@ -1926,7 +1926,7 @@ /* Define to `unsigned int' if does not define. */ #undef size_t -/* Define to `int' if does not define. */ +/* Define to `int' if or does not define. */ #undef socklen_t /* Define to `int' if doesn't define. */ From a2f900255125f608084ae8e14a055111cbd011ca Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 14 Feb 2023 20:40:42 +0530 Subject: [PATCH 015/123] Enable some windows specific modules --- Modules/Setup.bootstrap.in | 2 ++ Modules/Setup.stdlib.in | 7 +++++++ Modules/_winapi.c | 2 ++ PC/msvcrtmodule.c | 2 ++ Python/dynamic_annotations.c | 2 +- configure.ac | 10 ++++++++++ 6 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 3def7e8f810731..d8b41ee7f3b5e9 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -37,3 +37,5 @@ _symtable symtablemodule.c # build-in modules for windows platform: @USE_WIN32_MODULE@winreg ../PC/winreg.c +@MODULE_MSVCRT_TRUE@msvcrt -DPy_BUILD_CORE ../PC/msvcrtmodule.c +@MODULE__WINAPI_TRUE@_winapi _winapi.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 5e587dd3a5a397..98e3ef2fdf6a6f 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -160,6 +160,13 @@ # _scproxy needs SystemConfiguration and CoreFoundation framework @MODULE__SCPROXY_TRUE@_scproxy _scproxy.c +############################################################################ +# Windows specific modules + +@MODULE__OVERLAPPED_TRUE@_overlapped overlapped.c +@MODULE__MSI_TRUE@_msi ../PC/_msi.c +@MODULE_WINSOUND_TRUE@winsound ../PC/winsound.c + ############################################################################ # Test modules diff --git a/Modules/_winapi.c b/Modules/_winapi.c index ac3ee113ffda71..b5d14686d23c97 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -44,7 +44,9 @@ #endif #include "windows.h" #include +#if defined(Py_DEBUG) #include +#endif #include "winreparse.h" #if defined(MS_WIN32) && !defined(MS_WIN64) diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 53ef26b732f615..10cf78258a8084 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -22,7 +22,9 @@ #include #include #include +#ifdef _DEBUG #include +#endif #include #ifdef _MSC_VER diff --git a/Python/dynamic_annotations.c b/Python/dynamic_annotations.c index 7febaa09df1950..70d5b3dc72069c 100644 --- a/Python/dynamic_annotations.c +++ b/Python/dynamic_annotations.c @@ -27,7 +27,7 @@ * Author: Kostya Serebryany */ -#ifdef _MSC_VER +#ifdef MS_WINDOWS # include #endif diff --git a/configure.ac b/configure.ac index 37554f7facf8ea..8ccec5e9950f2f 100644 --- a/configure.ac +++ b/configure.ac @@ -7791,6 +7791,16 @@ PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) +dnl windows specific modules +PY_STDLIB_MOD([msvcrt], [test "$MACHDEP" = "win32"]) +PY_STDLIB_MOD([_winapi], [test "$MACHDEP" = "win32"]) +PY_STDLIB_MOD([_msi], [test "$MACHDEP" = "win32"], [], [], + [-lmsi -lcabinet -lrpcrt4]) +PY_STDLIB_MOD([winsound], [test "$MACHDEP" = "win32"], [], [], + [-lwinmm]) +PY_STDLIB_MOD([_overlapped], [test "$MACHDEP" = "win32"], [], [], + [-lws2_32]) + dnl test modules PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) From e8f2489cdc8d8b45c62ec0eff2856368bfb2bf3e Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Tue, 21 Sep 2021 20:52:42 +0200 Subject: [PATCH 016/123] sysconfig: MINGW build extensions with GCC --- Lib/sysconfig.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 7a7c0e20914e5b..ed24aebe3a83c1 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -800,6 +800,8 @@ def get_platform(): """ if os.name == 'nt': + if 'GCC' in sys.version: + return 'mingw' if 'amd64' in sys.version.lower(): return 'win-amd64' if '(arm)' in sys.version.lower(): From 0d4ec3ee1ae4efa88e9ac1a3215bd5314dcfcc5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:38 +0530 Subject: [PATCH 017/123] sysconfig: treat MINGW builds as POSIX builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Lib/sysconfig.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index ed24aebe3a83c1..e05a67d426cdd4 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -99,13 +99,18 @@ }, } +# GCC[mingw*] use posix build system +_POSIX_BUILD = os.name == 'posix' or \ + (os.name == "nt" and 'GCC' in sys.version) + # For the OS-native venv scheme, we essentially provide an alias: -if os.name == 'nt': +if os.name == 'nt' and not _POSIX_BUILD: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['nt_venv'] else: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] + # NOTE: site.py has copy of this function. # Sync it when modify this function. def _getuserbase(): @@ -120,7 +125,7 @@ def _getuserbase(): def joinuser(*args): return os.path.expanduser(os.path.join(*args)) - if os.name == "nt": + if os.name == "nt" and not _POSIX_BUILD: base = os.environ.get("APPDATA") or "~" return joinuser(base, "Python") @@ -288,7 +293,7 @@ def _expand_vars(scheme, vars): def _get_preferred_schemes(): - if os.name == 'nt': + if os.name == 'nt' and not _POSIX_BUILD: return { 'prefix': 'nt', 'home': 'posix_home', @@ -622,7 +627,7 @@ def parse_config_h(fp, vars=None): def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: - if os.name == "nt": + if os.name == "nt" and not _POSIX_BUILD: inc_dir = os.path.join(_PROJECT_BASE, "PC") else: inc_dir = _PROJECT_BASE @@ -690,10 +695,10 @@ def _init_config_vars(): except AttributeError: _CONFIG_VARS['py_version_nodot_plat'] = '' - if os.name == 'nt': + if os.name == 'nt' and not _POSIX_BUILD: _init_non_posix(_CONFIG_VARS) _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': + if _POSIX_BUILD: _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the @@ -703,7 +708,7 @@ def _init_config_vars(): # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) - if os.name == 'posix': + if _POSIX_BUILD: if _PYTHON_BUILD: # If srcdir is a relative path (typically '.' or '..') # then it should be interpreted relative to the directory From 3071b5192abc8a6f89f5652bf3263161ef136820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:39 +0530 Subject: [PATCH 018/123] Add support for stdcall without underscore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the case used in MINGW Co-authored-by: Алексей --- Modules/_ctypes/_ctypes.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index d3b496887b5692..c208223b3f14a5 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3416,6 +3416,18 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) mangled_name = alloca(strlen(name) + 1 + 1 + 1 + 3); /* \0 _ @ %d */ if (!mangled_name) return NULL; + /* Issue: for stdcall decorated export functions MSVC compiler adds + * underscore, but GCC compiler create them without. This is + * visible by example for _ctypes_test.pyd module. + * As well functions from system libraries are without underscore. + * Solutions: + * - If a python module is build with gcc option --add-stdcall-alias + * the module will contain XXX as alias for function XXX@ as result + * first search in this method will succeed. + * - Distutil may use compiler to create def-file, to modify it as + * add underscore alias and with new def file to create module. + * - Or may be just to search for function without underscore. + */ for (i = 0; i < 32; ++i) { sprintf(mangled_name, "_%s@%d", name, i*4); Py_BEGIN_ALLOW_THREADS @@ -3423,6 +3435,13 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) Py_END_ALLOW_THREADS if (address) return address; + /* search for function without underscore as weel */ + sprintf(mangled_name, "%s@%d", name, i*4); + Py_BEGIN_ALLOW_THREADS + address = (PPROC)GetProcAddress(handle, mangled_name); + Py_END_ALLOW_THREADS + if (address) + return address; } return NULL; #endif From dc7508ee3147501683362d2a1336ff809885c5a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:42 +0530 Subject: [PATCH 019/123] Customize site for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Lib/site.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index aed254ad504d35..fadfccf9f77377 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -89,6 +89,12 @@ USER_BASE = None +# Same as defined in Lib/sysconfig.py +# redeclared since sysconfig is large for site. +# GCC[mingw*] use posix build system +_POSIX_BUILD = os.name == 'posix' or \ + (os.name == "nt" and 'GCC' in sys.version) + def _trace(message): if sys.flags.verbose: print(message, file=sys.stderr) @@ -294,7 +300,7 @@ def _getuserbase(): def joinuser(*args): return os.path.expanduser(os.path.join(*args)) - if os.name == "nt": + if os.name == "nt" and not _POSIX_BUILD: base = os.environ.get("APPDATA") or "~" return joinuser(base, "Python") @@ -309,7 +315,7 @@ def joinuser(*args): def _get_path(userbase): version = sys.version_info - if os.name == 'nt': + if os.name == 'nt' and not _POSIX_BUILD: ver_nodot = sys.winver.replace('.', '') return f'{userbase}\\Python{ver_nodot}\\site-packages' @@ -382,7 +388,7 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) - if os.sep == '/': + if _POSIX_BUILD: libdirs = [sys.platlibdir] if sys.platlibdir != "lib": libdirs.append("lib") From a3da6c38d96be769365d9763c22df0f260f8687a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:43 +0530 Subject: [PATCH 020/123] add python config sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Misc/python-config.sh.in | 62 +++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 2602fe24c0402e..195c317ada3825 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -1,32 +1,44 @@ #!/bin/sh -# Keep this script in sync with python-config.in - exit_with_usage () { echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" - exit $1 + exit 1 } +# Really, python-config.py (and thus .sh) should be called directly, but +# sometimes software (e.g. GDB) calls python-config.sh as if it were the +# Python executable, passing python-config.py as the first argument. +# Work around that oddness by ignoring any .py passed as first arg. +case "$1" in + *.py) + shift + ;; +esac + if [ "$1" = "" ] ; then - exit_with_usage 1 + exit_with_usage fi # Returns the actual prefix where this script was installed to. installed_prefix () { - RESULT=$(dirname $(cd $(dirname "$1") && pwd -P)) - if which readlink >/dev/null 2>&1 ; then - if readlink -f "$RESULT" >/dev/null 2>&1; then - RESULT=$(readlink -f "$RESULT") - fi + local RESULT=$(dirname $(cd $(dirname "$1") && pwd -P)) + if [ $(which readlink) ] ; then + RESULT=$(readlink -f "$RESULT") + fi + # Since we don't know where the output from this script will end up + # we keep all paths in Windows-land since MSYS2 can handle that + # while native tools can't handle paths in MSYS2-land. + if [ "$OSTYPE" = "cygwin" ] || [ "$OSTYPE" = "msys" ] ; then + RESULT=$(cd "$RESULT" && pwd -W) fi echo $RESULT } prefix_real=$(installed_prefix "$0") -# Use sed to fix paths from their built-to locations to their installed-to +# Use sed to fix paths from their built-to locations to their installed to # locations. Keep prefix & exec_prefix using their original values in case # they are referenced in other configure variables, to prevent double # substitution, issue #22140. @@ -41,13 +53,17 @@ LIBM="@LIBM@" LIBC="@LIBC@" SYSLIBS="$LIBM $LIBC" ABIFLAGS="@ABIFLAGS@" +# Protect against lack of substitution. +if [ "$ABIFLAGS" = "@""ABIFLAGS""@" ] ; then + ABIFLAGS= +fi LIBS="@LIBPYTHON@ @LIBS@ $SYSLIBS" LIBS_EMBED="-lpython${VERSION}${ABIFLAGS} @LIBS@ $SYSLIBS" BASECFLAGS="@BASECFLAGS@" -LDLIBRARY="@LDLIBRARY@" OPT="@OPT@" PY_ENABLE_SHARED="@PY_ENABLE_SHARED@" LDVERSION="@LDVERSION@" +LDLIBRARY="@LDLIBRARY@" LIBDEST=${prefix_real}/lib/python${VERSION} LIBPL=$(echo "@LIBPL@" | sed "s#$prefix#$prefix_real#") SO="@EXT_SUFFIX@" @@ -61,7 +77,7 @@ for ARG in $* do case $ARG in --help) - exit_with_usage 0 + exit_with_usage ;; --embed) PY_EMBED=1 @@ -69,7 +85,7 @@ do --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--abiflags|--configdir) ;; *) - exit_with_usage 1 + exit_with_usage ;; esac done @@ -80,37 +96,37 @@ fi for ARG in "$@" do - case "$ARG" in + case $ARG in --prefix) - echo "$prefix_real" + echo -ne "$prefix_real" ;; --exec-prefix) - echo "$exec_prefix_real" + echo -ne "$exec_prefix_real " ;; --includes) - echo "$INCDIR $PLATINCDIR" + echo -ne "$INCDIR $PLATINCDIR" ;; --cflags) - echo "$INCDIR $PLATINCDIR $BASECFLAGS $CFLAGS $OPT" + echo -ne "$INCDIR $PLATINCDIR $BASECFLAGS $CFLAGS $OPT" ;; --libs) - echo "$LIBS" + echo -ne "$LIBS" ;; --ldflags) LIBPLUSED= if [ "$PY_ENABLE_SHARED" = "0" ] ; then LIBPLUSED="-L$LIBPL" fi - echo "$LIBPLUSED -L$libdir $LIBS" + echo -ne "$LIBPLUSED -L$libdir $LIBS " ;; --extension-suffix) - echo "$SO" + echo -ne "$SO " ;; --abiflags) - echo "$ABIFLAGS" + echo -ne "$ABIFLAGS " ;; --configdir) - echo "$LIBPL" + echo -ne "$LIBPL " ;; esac done From 608ce98d36932f2735a4e83d4c99f82c5beba599 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 17 Jun 2021 18:51:46 +0530 Subject: [PATCH 021/123] mingw: prefer unix sep if MSYSTEM environment variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter Co-authored-by: cat Co-authored-by: Naveen M K --- Include/pylifecycle.h | 6 ++ Lib/ntpath.py | 106 ++++++++++++++++++--------------- Modules/posixmodule.c | 2 + Python/initconfig.c | 2 +- Python/pathconfig.c | 135 ++++++++++++++++++++++++++++++++++++++++++ Python/traceback.c | 2 +- 6 files changed, 203 insertions(+), 50 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index e4c3b09c963fe8..50d8b76d74b610 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -21,6 +21,12 @@ PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); +PyAPI_FUNC(wchar_t) Py_GetSepW(const wchar_t *); +PyAPI_FUNC(char) Py_GetSepA(const char *); + +PyAPI_FUNC(void) Py_NormalizeSepsW(wchar_t *); +PyAPI_FUNC(void) Py_NormalizeSepsA(char *); + /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level * exit functions. diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 1bef630bd9c0ff..c4855ae11f36f0 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -11,9 +11,7 @@ curdir = '.' pardir = '..' extsep = '.' -sep = '\\' pathsep = ';' -altsep = '/' defpath = '.;C:\\bin' devnull = 'nul' @@ -23,6 +21,14 @@ import genericpath from genericpath import * +if sys.platform == "win32" and "MSYSTEM" in os.environ: + sep = '/' + altsep = '\\' +else: + sep = '\\' + altsep = '/' +bsep = str.encode(sep) +baltsep = str.encode(altsep) __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", @@ -35,9 +41,33 @@ def _get_bothseps(path): if isinstance(path, bytes): - return b'\\/' + return bsep+baltsep + else: + return sep+altsep + +def _get_sep(path): + if isinstance(path, bytes): + return bsep else: - return '\\/' + return sep + +def _get_altsep(path): + if isinstance(path, bytes): + return baltsep + else: + return altsep + +def _get_colon(path): + if isinstance(path, bytes): + return b':' + else: + return ':' + +def _get_unc_prefix(path): + if isinstance(path, bytes): + return b'\\\\?\\UNC\\' + else: + return '\\\\?\\UNC\\' # Normalize the case of a pathname and map slashes to backslashes. # Other normalizations (such as optimizing '../' away) are not done @@ -59,14 +89,14 @@ def normcase(s): return s if isinstance(s, bytes): encoding = sys.getfilesystemencoding() - s = s.decode(encoding, 'surrogateescape').replace('/', '\\') + s = s.decode(encoding, 'surrogateescape').replace(altsep, sep) s = _LCMapStringEx(_LOCALE_NAME_INVARIANT, _LCMAP_LOWERCASE, s) return s.encode(encoding, 'surrogateescape') else: return _LCMapStringEx(_LOCALE_NAME_INVARIANT, _LCMAP_LOWERCASE, - s.replace('/', '\\')) + s.replace(altsep, sep)) except ImportError: def normcase(s): """Normalize case of pathname. @@ -75,8 +105,8 @@ def normcase(s): """ s = os.fspath(s) if isinstance(s, bytes): - return os.fsencode(os.fsdecode(s).replace('/', '\\').lower()) - return s.replace('/', '\\').lower() + return os.fsencode(os.fsdecode(s).replace(altsep, sep).lower()) + return s.replace(altsep, sep).lower() # Return whether a path is absolute. @@ -88,14 +118,9 @@ def normcase(s): def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) - if isinstance(s, bytes): - sep = b'\\' - altsep = b'/' - colon_sep = b':\\' - else: - sep = '\\' - altsep = '/' - colon_sep = ':\\' + sep = _get_sep(s) + altsep = _get_altsep(s) + colon_sep = _get_colon(s) + sep s = s[:3].replace(altsep, sep) # Absolute: UNC, device, and paths with a drive and root. # LEGACY BUG: isabs("/x") should be false since the path has no drive. @@ -107,14 +132,9 @@ def isabs(s): # Join two (or more) paths. def join(path, *paths): path = os.fspath(path) - if isinstance(path, bytes): - sep = b'\\' - seps = b'\\/' - colon = b':' - else: - sep = '\\' - seps = '\\/' - colon = ':' + sep = _get_sep(path) + seps = _get_bothseps(path) + colon = _get_colon(path) try: if not paths: path[:0] + sep #23780: Ensure compatible data type even if p is null. @@ -189,18 +209,12 @@ def splitroot(p): splitroot('Windows/notepad') == ('', '', 'Windows/notepad') """ p = os.fspath(p) - if isinstance(p, bytes): - sep = b'\\' - altsep = b'/' - colon = b':' - unc_prefix = b'\\\\?\\UNC\\' - empty = b'' - else: - sep = '\\' - altsep = '/' - colon = ':' - unc_prefix = '\\\\?\\UNC\\' - empty = '' + sep = _get_sep(p) + altsep = _get_altsep(p) + colon = _get_colon(p) + unc_prefix = _get_unc_prefix(p) + empty = b'' if isinstance(p, bytes) else '' + normp = p.replace(altsep, sep) if normp[:1] == sep: if normp[1:2] == sep: @@ -258,9 +272,9 @@ def split(p): def splitext(p): p = os.fspath(p) if isinstance(p, bytes): - return genericpath._splitext(p, b'\\', b'/', b'.') + return genericpath._splitext(p, bsep, baltsep, b'.') else: - return genericpath._splitext(p, '\\', '/', '.') + return genericpath._splitext(p, sep, altsep, '.') splitext.__doc__ = genericpath._splitext.__doc__ @@ -528,14 +542,12 @@ def expandvars(path): def normpath(path): """Normalize path, eliminating double slashes, etc.""" path = os.fspath(path) + sep = _get_sep(path) + altsep = _get_altsep(path) if isinstance(path, bytes): - sep = b'\\' - altsep = b'/' curdir = b'.' pardir = b'..' else: - sep = '\\' - altsep = '/' curdir = '.' pardir = '..' path = path.replace(altsep, sep) @@ -773,6 +785,7 @@ def realpath(path, *, strict=False): # strip the prefix anyway. if ex.winerror == initial_winerror: path = spath + path = normpath(path) return path @@ -782,12 +795,11 @@ def realpath(path, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + sep = _get_sep(path) if isinstance(path, bytes): - sep = b'\\' curdir = b'.' pardir = b'..' else: - sep = '\\' curdir = '.' pardir = '..' @@ -842,13 +854,11 @@ def commonpath(paths): raise ValueError('commonpath() arg is an empty sequence') paths = tuple(map(os.fspath, paths)) + sep = _get_sep(paths[0]) + altsep = _get_altsep(paths[0]) if isinstance(paths[0], bytes): - sep = b'\\' - altsep = b'/' curdir = b'.' else: - sep = '\\' - altsep = '/' curdir = '.' try: diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 1a62ddd91d3b5e..2cb0c5bc6d361b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4006,6 +4006,7 @@ posix_getcwd(int use_bytes) return NULL; } + Py_NormalizeSepsW(wbuf2); PyObject *resobj = PyUnicode_FromWideChar(wbuf2, len); if (wbuf2 != wbuf) { PyMem_RawFree(wbuf2); @@ -4892,6 +4893,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) target_path = tmp; } + Py_NormalizeSepsW(target_path); result = PyUnicode_FromWideChar(target_path, result_length); if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); diff --git a/Python/initconfig.c b/Python/initconfig.c index ed4c4e23f34a80..6ba6f5bfc863b5 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -164,7 +164,7 @@ static const char usage_envvars[] = "PYTHONWARNINGS : warning control (-W)\n" ; -#if defined(MS_WINDOWS) +#if defined(_MSC_VER) # define PYTHONHOMEHELP "\\python{major}{minor}" #else # define PYTHONHOMEHELP "/lib/pythonX.X" diff --git a/Python/pathconfig.c b/Python/pathconfig.c index be0f97c4b204a9..3d684d68ff12df 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -18,6 +18,140 @@ extern "C" { #endif +#ifdef __MINGW32__ +#define wcstok wcstok_s +#include +#endif + +static int +Py_StartsWithA(const char * str, const char * prefix) +{ + while(*prefix) + { + if(*prefix++ != *str++) + return 0; + } + + return 1; +} + +static int +Py_StartsWithW(const wchar_t * str, const wchar_t * prefix) +{ + while(*prefix) + { + if(*prefix++ != *str++) + return 0; + } + + return 1; +} + +char +Py_GetSepA(const char *name) +{ + char* msystem = (char*)2; /* So that non Windows use / as sep */ + static char sep = '\0'; +#ifdef _WIN32 + /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx + * The "\\?\" prefix .. indicate that the path should be passed to the system with minimal + * modification, which means that you cannot use forward slashes to represent path separators + */ + if (name != NULL && Py_StartsWithA(name, "\\\\?\\") != 0) + { + return '\\'; + } +#endif + if (sep != '\0') + return sep; +#if defined(__MINGW32__) + msystem = Py_GETENV("MSYSTEM"); +#endif + if (msystem != NULL) + sep = '/'; + else + sep = '\\'; + return sep; +} + +static char +Py_GetAltSepA(const char *name) +{ + char sep = Py_GetSepA(name); + if (sep == '/') + return '\\'; + return '/'; +} + +void +Py_NormalizeSepsA(char *name) +{ + assert(name != NULL); + char sep = Py_GetSepA(name); + char altsep = Py_GetAltSepA(name); + char* seps; + if (name[0] != '\0' && name[1] == ':') { + name[0] = toupper(name[0]); + } + seps = strchr(name, altsep); + while(seps) { + *seps = sep; + seps = strchr(seps, altsep); + } +} + +wchar_t +Py_GetSepW(const wchar_t *name) +{ + char* msystem = (char*)2; /* So that non Windows use / as sep */ + static wchar_t sep = L'\0'; +#ifdef _WIN32 + /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx + * The "\\?\" prefix .. indicate that the path should be passed to the system with minimal + * modification, which means that you cannot use forward slashes to represent path separators + */ + if (name != NULL && Py_StartsWithW(name, L"\\\\?\\") != 0) + { + return L'\\'; + } +#endif + if (sep != L'\0') + return sep; +#if defined(__MINGW32__) + msystem = Py_GETENV("MSYSTEM"); +#endif + if (msystem != NULL) + sep = L'/'; + else + sep = L'\\'; + return sep; +} + +static wchar_t +Py_GetAltSepW(const wchar_t *name) +{ + char sep = Py_GetSepW(name); + if (sep == L'/') + return L'\\'; + return L'/'; +} + +void +Py_NormalizeSepsW(wchar_t *name) +{ + assert(name != NULL); + wchar_t sep = Py_GetSepW(name); + wchar_t altsep = Py_GetAltSepW(name); + wchar_t* seps; + if (name[0] != L'\0' && name[1] == L':') { + name[0] = towupper(name[0]); + } + seps = wcschr(name, altsep); + while(seps) { + *seps = sep; + seps = wcschr(seps, altsep); + } +} /* External interface */ @@ -317,6 +451,7 @@ _Py_SetProgramFullPath(const wchar_t *program_full_path) if (has_value && _Py_path_config.program_full_path == NULL) { path_out_of_memory(__func__); } + Py_NormalizeSepsW(_Py_path_config.program_name); } diff --git a/Python/traceback.c b/Python/traceback.c index 1cb82c57203758..37df982bb737f3 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -342,7 +342,7 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * filepath = PyBytes_AS_STRING(filebytes); /* Search tail of filename in sys.path before giving up */ - tail = strrchr(filepath, SEP); + tail = strrchr(filepath, Py_GetSepA(filepath)); if (tail == NULL) tail = filepath; else From 061c843266bf9f617e004ab9ee9f45425be47c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:50 +0530 Subject: [PATCH 022/123] msys cygwin semi native build sysconfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Lib/sysconfig.py | 8 +++++++ Makefile.pre.in | 7 ++++++ configure.ac | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index e05a67d426cdd4..60626d23b81365 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -451,6 +451,14 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): if isinstance(v, str): done[k] = v.strip() + # any keys that have one with the same name suffixed with _b2h + # need to be replaced with the value of the _b2h key. + # This converts from MSYS*/Cygwin paths to Windows paths. + for k, v in dict(done).items(): + if isinstance(k, str): + if k.endswith("_b2h"): + done[k[:-4]]=v + # save the results in the global dictionary vars.update(done) return vars diff --git a/Makefile.pre.in b/Makefile.pre.in index 5e005b63e94597..63ca7ad96a3f6c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -142,6 +142,13 @@ exec_prefix= @exec_prefix@ # Install prefix for data files datarootdir= @datarootdir@ +# Locations needed for semi-native fixup of sysconfig. +srcdir_b2h= @srcdir_b2h@ +VPATH_b2h= @VPATH_b2h@ +abs_srcdir_b2h= @abs_srcdir_b2h@ +abs_builddir_b2h= @abs_builddir_b2h@ +prefix_b2h= @prefix_b2h@ + # Expanded directories BINDIR= @bindir@ LIBDIR= @libdir@ diff --git a/configure.ac b/configure.ac index 8ccec5e9950f2f..c8407ad0472474 100644 --- a/configure.ac +++ b/configure.ac @@ -755,6 +755,65 @@ then [Define to include mbstate_t for mbrtowc]) fi +# On 'semi-native' build systems (MSYS*/Cygwin targeting MinGW-w64) +# _sysconfigdata.py will contain paths that are correct only in the +# build environment. This means external modules will fail to build +# without setting up the same env and also that the build of Python +# itself will fail as the paths are not correct for the host tools. +# +# Also, getpath.c uses GetModuleFileNameW (replacing \ with /) and +# compares that with the define VPATH (passed in via command-line) +# to determine whether it's the build- or the installed-Python. +# +# To work around these issues a set of _b2h variables are created: +# VPATH_b2h, prefix_b2h, srcdir_b2h, abs_srcdir_b2h +# and abs_builddir_b2h +# .. where b2h stands for build to host. sysconfig.py replaces path +# prefixes matching the non-b2h versions with the b2h equivalents. +# +# (note this assumes the host compilers are native and *not* cross +# - in the 'semi-native' scenario only that is.) + +AC_DEFUN([ABS_PATH_HOST], +[$1=$(cd $$2 && pwd) + case $build_os in + mingw*) + case $host_os in + mingw*) $1=$(cd $$2 && pwd -W) ;; + *) ;; + esac + ;; + cygwin*) + case $host_os in + mingw*) $1=$(cygpath -w -m $$2) ;; + *) ;; + esac + ;; + esac +AC_SUBST([$1]) +]) + +AC_MSG_CHECKING(absolute host location of VPATH) +ABS_PATH_HOST([VPATH_b2h],[srcdir]) +AC_MSG_RESULT([$VPATH_b2h]) + +AC_MSG_CHECKING(absolute host location of prefix) +ABS_PATH_HOST([prefix_b2h],[prefix]) +AC_MSG_RESULT([$prefix_b2h]) + +AC_MSG_CHECKING(absolute host location of srcdir) +ABS_PATH_HOST([srcdir_b2h],[srcdir]) +AC_MSG_RESULT([$srcdir_b2h]) + +AC_MSG_CHECKING(absolute host location of abs_srcdir) +ABS_PATH_HOST([abs_srcdir_b2h],[srcdir]) +AC_MSG_RESULT([$abs_srcdir_b2h]) + +my_builddir=. +AC_MSG_CHECKING(Absolute host location of abs_builddir) +ABS_PATH_HOST([abs_builddir_b2h],[my_builddir]) +AC_MSG_RESULT([$abs_builddir_b2h]) + AC_MSG_CHECKING([for init system calls]) AC_SUBST(INITSYS) case $host in From 99a6cac7eb223d6810f79f8cc59426efee373bf4 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Tue, 21 Sep 2021 20:53:59 +0200 Subject: [PATCH 023/123] sysconfig: mingw sysconfig like posix --- Lib/sysconfig.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 60626d23b81365..a0e07b93be6bf6 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -49,13 +49,13 @@ 'data': '{base}', }, 'nt': { - 'stdlib': '{installed_base}/Lib', - 'platstdlib': '{base}/Lib', - 'purelib': '{base}/Lib/site-packages', - 'platlib': '{base}/Lib/site-packages', - 'include': '{installed_base}/Include', - 'platinclude': '{installed_base}/Include', - 'scripts': '{base}/Scripts', + 'stdlib': '{installed_base}/lib/python{py_version_short}', + 'platstdlib': '{base}/lib/python{py_version_short}', + 'purelib': '{base}/lib/python{py_version_short}/site-packages', + 'platlib': '{base}/lib/python{py_version_short}/site-packages', + 'include': '{installed_base}/include/python{py_version_short}', + 'platinclude': '{installed_base}/include/python{py_version_short}', + 'scripts': '{base}/bin', 'data': '{base}', }, # Downstream distributors can overwrite the default install scheme. @@ -141,12 +141,12 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'stdlib': '{userbase}/lib/python{py_version_short}', + 'platstdlib': '{userbase}/lib/python{py_version_short}', + 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', + 'include': '{userbase}/include/python{py_version_short}', + 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'posix_user': { @@ -589,7 +589,7 @@ def _init_non_posix(vars): except IndexError: pass vars['EXE'] = '.exe' - vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT + vars['VERSION'] = _PY_VERSION_SHORT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) vars['TZPATH'] = '' From e0ac354606100754d608db98b5e7334f0c729bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:53 +0530 Subject: [PATCH 024/123] mingw use backslashes in compileall py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Lib/compileall.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/compileall.py b/Lib/compileall.py index d394156cedc4e7..ef3633e9537615 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -38,6 +38,8 @@ def _walk_dir(dir, maxlevels, quiet=0): if name == '__pycache__': continue fullname = os.path.join(dir, name) + if sys.platform == "win32" and sys.version.find("GCC") >= 0: + fullname = fullname.replace('\\','/') if not os.path.isdir(fullname): yield fullname elif (maxlevels > 0 and name != os.curdir and name != os.pardir and From 73e185f320f38a2f48eb6a3fe7c56b67697dd2f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:55 +0530 Subject: [PATCH 025/123] mingw pdcurses_ISPAD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/py_curses.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Include/py_curses.h b/Include/py_curses.h index e46b08e9cc414e..34228fbba55c81 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -36,6 +36,13 @@ #include #endif +#if defined(__MINGW32__) +#include +#if !defined(_ISPAD) +#define _ISPAD 0x10 +#endif +#endif + #ifdef HAVE_NCURSES_H /* configure was checking , but we will use , which has some or all these features. */ From b78fb154797801eb8136c24fbcbcede06f0b9438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:59 +0530 Subject: [PATCH 026/123] remove_path_max.default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/osdefs.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/osdefs.h b/Include/osdefs.h index 3243944a1483e9..99d49777a69d9a 100644 --- a/Include/osdefs.h +++ b/Include/osdefs.h @@ -10,7 +10,6 @@ extern "C" { #ifdef MS_WINDOWS #define SEP L'\\' #define ALTSEP L'/' -#define MAXPATHLEN 256 #define DELIM L';' #endif From a872b0c721d9a59f2c4bce13a367d1a8bd0c4b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:00 +0530 Subject: [PATCH 027/123] dont link with gettext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Modules/_localemodule.c | 7 +++++++ configure.ac | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index db8194372dae49..416bd591ea2664 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -12,6 +12,13 @@ This software comes with no warranty. Use at your own risk. #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_fileutils.h" +#ifdef __MINGW32__ +/* The header libintl.h and library libintl may exist on mingw host. + * To be compatible with MSVC build we has to undef some defines. + */ +#undef HAVE_LIBINTL_H +#undef HAVE_BIND_TEXTDOMAIN_CODESET +#endif #include #include diff --git a/configure.ac b/configure.ac index c8407ad0472474..cc7dc15d11f870 100644 --- a/configure.ac +++ b/configure.ac @@ -3878,10 +3878,19 @@ AC_SEARCH_LIBS([sem_init], [pthread rt posix4]) fi # check if we need libintl for locale functions +case $host in + *-*-mingw*) + dnl Native windows build don't use libintl (see _localemodule.c). + dnl Also we don't like setup.py to add "intl" library to the list + dnl when build _locale module. + ;; + *) AC_CHECK_LIB([intl], [textdomain], [AC_DEFINE([WITH_LIBINTL], [1], [Define to 1 if libintl is needed for locale functions.]) LIBS="-lintl $LIBS"]) + ;; +esac # checks for system dependent C++ extensions support case "$ac_sys_system" in From 8f7da60248ace9b0ebe7b160ea542004e7d51e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:01 +0530 Subject: [PATCH 028/123] ctypes python dll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Lib/ctypes/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 6cedee74236ea5..7fceeb8fa4e320 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -464,7 +464,9 @@ def LoadLibrary(self, name): cdll = LibraryLoader(CDLL) pydll = LibraryLoader(PyDLL) -if _os.name == "nt": +if _os.name == "nt" and _sys.version.find('GCC') >= 0: + pythonapi = PyDLL("libpython%d.%d%s.dll" % (_sys.version_info[:2] + (_sys.abiflags,)), None) +elif _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) From dff5e634a9abdd7e90e8f2216c1c1f21a9e2191a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:02 +0530 Subject: [PATCH 029/123] gdbm module includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Modules/_gdbmmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index ffd7b6420894b2..b536cedffce2a4 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -18,7 +18,7 @@ #include #include -#if defined(WIN32) && !defined(__CYGWIN__) +#if defined(WIN32) && !defined(__CYGWIN__) && !defined(__MINGW32__) #include "gdbmerrno.h" extern const char * gdbm_strerror(gdbm_error); #endif From 86525e289566459cd21d780f22676abcbaecacf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:03 +0530 Subject: [PATCH 030/123] use gnu_printf in format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/bytesobject.h | 4 ++-- Include/pyerrors.h | 4 ++-- Include/pyport.h | 6 ++++++ Include/sysmodule.h | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index ee448cd02bdab3..d10ec3f86cb696 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -35,9 +35,9 @@ PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyBytes_FromString(const char *); PyAPI_FUNC(PyObject *) PyBytes_FromObject(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list) - Py_GCC_ATTRIBUTE((format(printf, 1, 0))); + Py_PRINTF(1, 0); PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_PRINTF(1, 2); PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *); PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int); diff --git a/Include/pyerrors.h b/Include/pyerrors.h index d089fa71779330..f8e604eef546e8 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -321,9 +321,9 @@ PyAPI_FUNC(int) PyUnicodeTranslateError_SetReason( ); PyAPI_FUNC(int) PyOS_snprintf(char *str, size_t size, const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 3, 4))); + Py_PRINTF(3, 4); PyAPI_FUNC(int) PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) - Py_GCC_ATTRIBUTE((format(printf, 3, 0))); + Py_PRINTF(3, 0); #ifndef Py_LIMITED_API # define Py_CPYTHON_ERRORS_H diff --git a/Include/pyport.h b/Include/pyport.h index ddff496ba80544..df363d0daa2dbd 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -666,6 +666,12 @@ extern char * _getpty(int *, int, mode_t, int); #define Py_VA_COPY va_copy +#if defined(__GNUC__) && ((__GNUC__ == 4 && __GNUC_MINOR__>= 4) || __GNUC__ > 4) +# define Py_PRINTF(X,Y) Py_GCC_ATTRIBUTE((format(gnu_printf,X,Y))) +#else +# define Py_PRINTF(X,Y) Py_GCC_ATTRIBUTE((format(printf,X,Y))) +#endif + /* * Convenient macros to deal with endianness of the platform. WORDS_BIGENDIAN is * detected by configure and defined in pyconfig.h. The code in pyconfig.h diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 96f883870b34dc..55c558a9037b8e 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -15,9 +15,9 @@ Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int); Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetPath(const wchar_t *); PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_PRINTF(1, 2); PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_PRINTF(1, 2); PyAPI_FUNC(void) PySys_FormatStdout(const char *format, ...); PyAPI_FUNC(void) PySys_FormatStderr(const char *format, ...); From 93979947206bda423102ee12c28b2b26f0d1f9c0 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 17 Jun 2021 18:52:03 +0530 Subject: [PATCH 031/123] mingw fix ssl dont use enum_certificates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Lib/ssl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index 42ebb8ed38466d..791d5c548abe9c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -254,7 +254,7 @@ class _TLSMessageType: CHANGE_CIPHER_SPEC = 0x0101 -if sys.platform == "win32": +if sys.platform == "win32" and sys.version.find("GCC") == -1: from _ssl import enum_certificates, enum_crls from socket import socket, SOCK_STREAM, create_connection @@ -528,7 +528,7 @@ def _load_windows_store_certs(self, storename, purpose): def load_default_certs(self, purpose=Purpose.SERVER_AUTH): if not isinstance(purpose, _ASN1Object): raise TypeError(purpose) - if sys.platform == "win32": + if sys.platform == "win32" and sys.version.find("GCC") == -1: for storename in self._windows_cert_stores: self._load_windows_store_certs(storename, purpose) self.set_default_verify_paths() From be3f6f4ff434872c67ce58df9285c714d9e8fbe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:06 +0530 Subject: [PATCH 032/123] fix using dllhandle and winver mingw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Python/sysmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6ed28ec5fbd8ba..0182569172b9dd 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -45,7 +45,7 @@ Data members: # include #endif /* MS_WINDOWS */ -#ifdef MS_COREDLL +#if defined(MS_WINDOWS) && defined(Py_ENABLE_SHARED) extern void *PyWin_DLLhModule; /* A string loaded from the DLL at startup: */ extern const char *PyWin_DLLVersionString; @@ -3429,7 +3429,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS_FROM_STRING("byteorder", "little"); #endif -#ifdef MS_COREDLL +#if defined(MS_WINDOWS) && defined(Py_ENABLE_SHARED) SET_SYS("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule)); SET_SYS_FROM_STRING("winver", PyWin_DLLVersionString); #endif From 28bfd9dcda9d2c12ff9f2cdb9c34098ced453e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:07 +0530 Subject: [PATCH 033/123] Add AMD64 to sys config so msvccompiler get_build_version works --- Python/getcompiler.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Python/getcompiler.c b/Python/getcompiler.c index a5d26239e8772e..3e31c80bdf4fb1 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -10,7 +10,22 @@ #if defined(__clang__) #define COMPILER "[Clang " __clang_version__ "]" #elif defined(__GNUC__) -#define COMPILER "[GCC " __VERSION__ "]" +/* To not break compatibility with things that determine + CPU arch by calling get_build_version in msvccompiler.py + (such as NumPy) add "32 bit" or "64 bit (AMD64)" on Windows + and also use a space as a separator rather than a newline. */ +#if defined(_WIN32) +#define COMP_SEP " " +#if defined(__x86_64__) +#define ARCH_SUFFIX " 64 bit (AMD64)" +#else +#define ARCH_SUFFIX " 32 bit" +#endif +#else +#define COMP_SEP "\n" +#define ARCH_SUFFIX "" +#endif +#define COMPILER COMP_SEP "[GCC " __VERSION__ ARCH_SUFFIX "]" // Generic fallbacks. #elif defined(__cplusplus) #define COMPILER "[C++]" From 013c102dd2fb35ff2c4036689f6b12c0bd55251a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:08 +0530 Subject: [PATCH 034/123] MINGW link with additional library --- configure.ac | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configure.ac b/configure.ac index cc7dc15d11f870..7002b78672169a 100644 --- a/configure.ac +++ b/configure.ac @@ -7104,6 +7104,15 @@ for dir in $SRCDIRS; do done AC_MSG_RESULT([done]) +# For mingw build need additional library for linking +case $host in + *-*-mingw*) + LIBS="$LIBS -lversion -lshlwapi -lpathcch" + ;; + *) + ;; +esac + # Availability of -O2: AC_CACHE_CHECK([for -O2], [ac_cv_compile_o2], [ saved_cflags="$CFLAGS" From e1e7676fb6ed5732d83d13d3355c2bd82e1f37aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C3=BAl=20Ibarra=20Corretg=C3=A9?= Date: Thu, 17 Jun 2021 18:52:09 +0530 Subject: [PATCH 035/123] install msilib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter Co-authored-by: Ray Donnelly --- Makefile.pre.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 63ca7ad96a3f6c..5cc3d6dccae9ee 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2122,6 +2122,7 @@ LIBSUBDIRS= asyncio \ tkinter \ tomllib \ turtledemo \ + msilib \ unittest \ urllib \ venv venv/scripts venv/scripts/common venv/scripts/posix \ From 550b26d84263d23e198d5689342b0deeb8a45a34 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:10 +0530 Subject: [PATCH 036/123] fix signal module build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Modules/Setup.bootstrap.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index d8b41ee7f3b5e9..ec0f81eea2f4ff 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -9,7 +9,7 @@ atexit atexitmodule.c faulthandler faulthandler.c @INITSYS@ posixmodule.c -_signal signalmodule.c +_signal signalmodule.c -lws2_32 _tracemalloc _tracemalloc.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig From cfdd9f74aefca9694fc28169dfdef2bf40d187b5 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:11 +0530 Subject: [PATCH 037/123] build: build winconsoleio and _testconsole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Modules/Setup.bootstrap.in | 2 +- Modules/Setup.stdlib.in | 1 + configure.ac | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index ec0f81eea2f4ff..cec80f5a9f0f19 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -16,7 +16,7 @@ _tracemalloc _tracemalloc.c _codecs _codecsmodule.c _collections _collectionsmodule.c errno errnomodule.c -_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c +_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c _io/winconsoleio.c itertools itertoolsmodule.c _sre _sre/sre.c _thread _threadmodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 98e3ef2fdf6a6f..b96db5d6d2b1e9 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -184,6 +184,7 @@ @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c @MODULE__TESTMULTIPHASE_TRUE@_testsinglephase _testsinglephase.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c +@MODULE__TESTCONSOLE_TRUE@_testconsole ../PC/_testconsole.c # Limited API template modules; must be built as shared modules. @MODULE_XXLIMITED_TRUE@xxlimited xxlimited.c diff --git a/configure.ac b/configure.ac index 7002b78672169a..1c8d1e73a7289e 100644 --- a/configure.ac +++ b/configure.ac @@ -7887,6 +7887,7 @@ PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testconsole], [test "$TEST_MODULES" = yes -a "$MACHDEP" = "win32"]) PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes], [], [$LIBM]) From 4276906624dea69f272fef17da4422f7251db687 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:12 +0530 Subject: [PATCH 038/123] expose sem_unlink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Modules/_multiprocessing/multiprocessing.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 8f9daa5c3de0cc..eb2cb5f86f2c9b 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -180,7 +180,7 @@ static PyMethodDef module_methods[] = { _MULTIPROCESSING_RECV_METHODDEF _MULTIPROCESSING_SEND_METHODDEF #endif -#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) +#if defined(MS_WINDOWS) || (!defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__)) _MULTIPROCESSING_SEM_UNLINK_METHODDEF #endif {NULL} From 2d45102621b58dd47a8bb5a14687689e090379bb Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:14 +0530 Subject: [PATCH 039/123] Use cygpty while using isatty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit so that msys shells are detected as interactive ones Co-authored-by: Алексей Co-authored-by: Naveen M K --- Include/iscygpty.h | 41 ++++++++++ Makefile.pre.in | 2 + Modules/_io/fileio.c | 3 +- Modules/main.c | 3 +- Modules/posixmodule.c | 3 +- Objects/fileobject.c | 3 +- Python/bltinmodule.c | 5 +- Python/fileutils.c | 7 +- Python/frozenmain.c | 3 +- Python/iscygpty.c | 185 ++++++++++++++++++++++++++++++++++++++++++ Python/pylifecycle.c | 5 +- 11 files changed, 248 insertions(+), 12 deletions(-) create mode 100644 Include/iscygpty.h create mode 100644 Python/iscygpty.c diff --git a/Include/iscygpty.h b/Include/iscygpty.h new file mode 100644 index 00000000000000..82fd0affbd4875 --- /dev/null +++ b/Include/iscygpty.h @@ -0,0 +1,41 @@ +/* + * iscygpty.h -- part of ptycheck + * https://github.com/k-takata/ptycheck + * + * Copyright (c) 2015-2017 K.Takata + * + * You can redistribute it and/or modify it under the terms of either + * the MIT license (as described below) or the Vim license. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _ISCYGPTY_H +#define _ISCYGPTY_H + +#ifdef _WIN32 +int is_cygpty(int fd); +int is_cygpty_used(void); +#else +#define is_cygpty(fd) 0 +#define is_cygpty_used() 0 +#endif + +#endif /* _ISCYGPTY_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 5cc3d6dccae9ee..2af2beea09ecc7 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -406,6 +406,7 @@ PYTHON_OBJS= \ Python/instrumentation.o \ Python/intrinsics.o \ Python/legacy_tracing.o \ + Python/iscygpty.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ @@ -1636,6 +1637,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/import.h \ $(srcdir)/Include/interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ + $(srcdir)/Include/iscygpty.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ $(srcdir)/Include/longobject.h \ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index bab68077a2144a..919b8b0429e84b 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -20,6 +20,7 @@ #endif #include /* For offsetof */ #include "_iomodule.h" +#include "iscygpty.h" /* * Known likely problems: @@ -1143,7 +1144,7 @@ _io_FileIO_isatty_impl(fileio *self) return err_closed(); Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - res = isatty(self->fd); + res = isatty(self->fd) || is_cygpty(self->fd); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS return PyBool_FromLong(res); diff --git a/Modules/main.c b/Modules/main.c index f4e4018212f59e..fbdccddaee955c 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -7,6 +7,7 @@ #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "iscygpty.h" /* Includes for exit_sigint() */ #include // perror() @@ -92,7 +93,7 @@ static inline int config_run_code(const PyConfig *config) static int stdin_is_interactive(const PyConfig *config) { - return (isatty(fileno(stdin)) || config->interactive); + return (isatty(fileno(stdin)) || config->interactive || is_cygpty(fileno(stdin))); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2cb0c5bc6d361b..51b3e525403039 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -54,6 +54,7 @@ #ifdef __ANDROID__ # undef HAVE_FACCESSAT #endif +#include "iscygpty.h" #include // ctermid() #include // system() @@ -11235,7 +11236,7 @@ os_isatty_impl(PyObject *module, int fd) int return_value; Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - return_value = isatty(fd); + return_value = isatty(fd) || is_cygpty(fd); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS return return_value; diff --git a/Objects/fileobject.c b/Objects/fileobject.c index e99e155f2b8c98..c74d2af4903a55 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -3,6 +3,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "iscygpty.h" #include "pycore_runtime.h" // _PyRuntime #if defined(HAVE_GETC_UNLOCKED) && !defined(_Py_MEMORY_SANITIZER) @@ -382,7 +383,7 @@ stdprinter_isatty(PyStdPrinter_Object *self, PyObject *Py_UNUSED(ignored)) } Py_BEGIN_ALLOW_THREADS - res = isatty(self->fd); + res = isatty(self->fd) || is_cygpty(self->fd); Py_END_ALLOW_THREADS return PyBool_FromLong(res); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2ae28693bee05c..d918964a90e4b1 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1,6 +1,7 @@ /* Built-in functions */ #include "Python.h" +#include "iscygpty.h" #include #include "pycore_ast.h" // _PyAST_Validate() #include "pycore_call.h" // _PyObject_CallNoArgs() @@ -2163,7 +2164,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) if (fd < 0 && PyErr_Occurred()) { goto error; } - tty = fd == fileno(stdin) && isatty(fd); + tty = fd == fileno(stdin) && (isatty(fd) || is_cygpty(fd)); } if (tty) { tmp = PyObject_CallMethodNoArgs(fout, &_Py_ID(fileno)); @@ -2176,7 +2177,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) Py_DECREF(tmp); if (fd < 0 && PyErr_Occurred()) goto error; - tty = fd == fileno(stdout) && isatty(fd); + tty = fd == fileno(stdout) && (isatty(fd) || is_cygpty(fd)); } } diff --git a/Python/fileutils.c b/Python/fileutils.c index c7521751cd26d1..6d3b04ff5bb9e6 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "iscygpty.h" #include "pycore_fileutils.h" // fileutils definitions #include "pycore_runtime.h" // _PyRuntime #include "osdefs.h" // SEP @@ -77,7 +78,7 @@ _Py_device_encoding(int fd) int valid; Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - valid = isatty(fd); + valid = isatty(fd) || is_cygpty(fd); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS if (!valid) @@ -1925,12 +1926,12 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) depending on heap usage). */ if (gil_held) { Py_BEGIN_ALLOW_THREADS - if (isatty(fd)) { + if (isatty(fd) || is_cygpty(fd)) { count = 32767; } Py_END_ALLOW_THREADS } else { - if (isatty(fd)) { + if (isatty(fd) || is_cygpty(fd)) { count = 32767; } } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index f8be165f7671df..d18c7b3884c27b 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_runtime.h" // _PyRuntime_Initialize() #include +#include "iscygpty.h" #ifdef MS_WINDOWS extern void PyWinFreeze_ExeInit(void); @@ -71,7 +72,7 @@ Py_FrozenMain(int argc, char **argv) sts = 0; } - if (inspect && isatty((int)fileno(stdin))) { + if (inspect && (isatty((int)fileno(stdin)) || is_cygpty((int)fileno(stdin)))) sts = PyRun_AnyFile(stdin, "") != 0; } diff --git a/Python/iscygpty.c b/Python/iscygpty.c new file mode 100644 index 00000000000000..722f88f2f46dcb --- /dev/null +++ b/Python/iscygpty.c @@ -0,0 +1,185 @@ +/* + * iscygpty.c -- part of ptycheck + * https://github.com/k-takata/ptycheck + * + * Copyright (c) 2015-2017 K.Takata + * + * You can redistribute it and/or modify it under the terms of either + * the MIT license (as described below) or the Vim license. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifdef _WIN32 + +#include +#include +#include +#include + +#ifdef USE_FILEEXTD +/* VC 7.1 or earlier doesn't support SAL. */ +# if !defined(_MSC_VER) || (_MSC_VER < 1400) +# define __out +# define __in +# define __in_opt +# endif +/* Win32 FileID API Library: + * http://www.microsoft.com/en-us/download/details.aspx?id=22599 + * Needed for WinXP. */ +# include +#else /* USE_FILEEXTD */ +/* VC 8 or earlier. */ +# if defined(_MSC_VER) && (_MSC_VER < 1500) +# ifdef ENABLE_STUB_IMPL +# define STUB_IMPL +# else +# error "Win32 FileID API Library is required for VC2005 or earlier." +# endif +# endif +#endif /* USE_FILEEXTD */ + + +#include "iscygpty.h" + +//#define USE_DYNFILEID +#ifdef USE_DYNFILEID +typedef BOOL (WINAPI *pfnGetFileInformationByHandleEx)( + HANDLE hFile, + FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + LPVOID lpFileInformation, + DWORD dwBufferSize +); +static pfnGetFileInformationByHandleEx pGetFileInformationByHandleEx = NULL; + +# ifndef USE_FILEEXTD +static BOOL WINAPI stub_GetFileInformationByHandleEx( + HANDLE hFile, + FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + LPVOID lpFileInformation, + DWORD dwBufferSize + ) +{ + return FALSE; +} +# endif + +static void setup_fileid_api(void) +{ + if (pGetFileInformationByHandleEx != NULL) { + return; + } + pGetFileInformationByHandleEx = (pfnGetFileInformationByHandleEx) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), + "GetFileInformationByHandleEx"); + if (pGetFileInformationByHandleEx == NULL) { +# ifdef USE_FILEEXTD + pGetFileInformationByHandleEx = GetFileInformationByHandleEx; +# else + pGetFileInformationByHandleEx = stub_GetFileInformationByHandleEx; +# endif + } +} +#else +# define pGetFileInformationByHandleEx GetFileInformationByHandleEx +# define setup_fileid_api() +#endif + + +#define is_wprefix(s, prefix) \ + (wcsncmp((s), (prefix), sizeof(prefix) / sizeof(WCHAR) - 1) == 0) + +/* Check if the fd is a cygwin/msys's pty. */ +int is_cygpty(int fd) +{ +#ifdef STUB_IMPL + return 0; +#else + HANDLE h; + int size = sizeof(FILE_NAME_INFO) + sizeof(WCHAR) * (MAX_PATH - 1); + FILE_NAME_INFO *nameinfo; + WCHAR *p = NULL; + + setup_fileid_api(); + + h = (HANDLE) _get_osfhandle(fd); + if (h == INVALID_HANDLE_VALUE) { + return 0; + } + /* Cygwin/msys's pty is a pipe. */ + if (GetFileType(h) != FILE_TYPE_PIPE) { + return 0; + } + nameinfo = malloc(size + sizeof(WCHAR)); + if (nameinfo == NULL) { + return 0; + } + /* Check the name of the pipe: + * '\{cygwin,msys}-XXXXXXXXXXXXXXXX-ptyN-{from,to}-master' */ + if (pGetFileInformationByHandleEx(h, FileNameInfo, nameinfo, size)) { + nameinfo->FileName[nameinfo->FileNameLength / sizeof(WCHAR)] = L'\0'; + p = nameinfo->FileName; + if (is_wprefix(p, L"\\cygwin-")) { /* Cygwin */ + p += 8; + } else if (is_wprefix(p, L"\\msys-")) { /* MSYS and MSYS2 */ + p += 6; + } else { + p = NULL; + } + if (p != NULL) { + while (*p && isxdigit(*p)) /* Skip 16-digit hexadecimal. */ + ++p; + if (is_wprefix(p, L"-pty")) { + p += 4; + } else { + p = NULL; + } + } + if (p != NULL) { + while (*p && isdigit(*p)) /* Skip pty number. */ + ++p; + if (is_wprefix(p, L"-from-master")) { + //p += 12; + } else if (is_wprefix(p, L"-to-master")) { + //p += 10; + } else { + p = NULL; + } + } + } + free(nameinfo); + return (p != NULL); +#endif /* STUB_IMPL */ +} + +/* Check if at least one cygwin/msys pty is used. */ +int is_cygpty_used(void) +{ + int fd, ret = 0; + + for (fd = 0; fd < 3; fd++) { + ret |= is_cygpty(fd); + } + return ret; +} + +#endif /* _WIN32 */ + +/* vim: set ts=4 sw=4: */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f5bea14b16b401..0bbb279511a3e5 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -30,6 +30,7 @@ #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "opcode.h" +#include "iscygpty.h" #include // setlocale() #include // getenv() @@ -3113,7 +3114,7 @@ Py_Exit(int sts) int Py_FdIsInteractive(FILE *fp, const char *filename) { - if (isatty(fileno(fp))) { + if (isatty(fileno(fp)) || is_cygpty(fileno(fp))) { return 1; } if (!_Py_GetConfig()->interactive) { @@ -3128,7 +3129,7 @@ Py_FdIsInteractive(FILE *fp, const char *filename) int _Py_FdIsInteractive(FILE *fp, PyObject *filename) { - if (isatty(fileno(fp))) { + if (isatty(fileno(fp)) || is_cygpty(fileno(fp))) { return 1; } if (!_Py_GetConfig()->interactive) { From 07aece0a50147865eb99d0f786823044d73c371a Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:16 +0530 Subject: [PATCH 040/123] build: link win resource files and build pythonw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Makefile.pre.in | 26 +++++++++++++++++++++----- configure.ac | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2af2beea09ecc7..d7916604dfc4c2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -38,6 +38,7 @@ CC= @CC@ CXX= @CXX@ LINKCC= @LINKCC@ AR= @AR@ +WINDRES= @WINDRES@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ @@ -124,6 +125,7 @@ PY_CORE_CFLAGS= $(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE PY_CORE_LDFLAGS=$(PY_LDFLAGS) $(PY_LDFLAGS_NODIST) # Strict or non-strict aliasing flags used to compile dtoa.c, see above CFLAGS_ALIASING=@CFLAGS_ALIASING@ +RCFLAGS=@RCFLAGS@ # Machine-dependent subdirectories @@ -286,6 +288,7 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) +BUILDPYTHONW= pythonw$(BUILDEXE) HOSTRUNNER= @HOSTRUNNER@ @@ -622,7 +625,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: all .PHONY: build_all -build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ +build_all: check-clean-src $(BUILDPYTHON) $(BUILDPYTHONW) platform sharedmods \ gdbhooks Programs/_testembed scripts checksharedmods rundsymutil .PHONY: build_wasm @@ -789,9 +792,21 @@ coverage-report: regen-token regen-frozen clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir) +python_exe.o: $(srcdir)/PC/python_exe.rc + $(WINDRES) $(RCFLAGS) -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_exe.rc $@ + +pythonw_exe.o: $(srcdir)/PC/pythonw_exe.rc + $(WINDRES) $(RCFLAGS) -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pythonw_exe.rc $@ + +python_nt.o: $(srcdir)/PC/python_nt.rc + $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ + +$(BUILDPYTHONW): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) pythonw_exe.o + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -mwindows -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) pythonw_exe.o + # Build the interpreter -$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) - $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) +$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) python_exe.o + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) python_exe.o platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -880,10 +895,10 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ # This rule builds the Cygwin Python DLL and import library if configured # for a shared core library; otherwise, this rule is a noop. -$(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) +$(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) python_nt.o if test -n "$(DLLLIBRARY)"; then \ $(LDSHARED) -Wl,--out-implib=$@ -o $(DLLLIBRARY) $^ \ - $(LIBS) $(MODLIBS) $(SYSLIBS); \ + $(LIBS) $(MODLIBS) $(SYSLIBS) python_nt.o; \ else true; \ fi @@ -1979,6 +1994,7 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ done if test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(BINDIR)/python3w$(EXE); \ else \ $(INSTALL_PROGRAM) $(STRIPFLAG) Mac/pythonw $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ fi diff --git a/configure.ac b/configure.ac index 1c8d1e73a7289e..af1f71173bd3bc 100644 --- a/configure.ac +++ b/configure.ac @@ -1762,6 +1762,10 @@ AC_SUBST([LINK_PYTHON_OBJS]) AC_SUBST([AR]) AC_CHECK_TOOLS([AR], [ar aal], [ar]) +# windres program +AC_SUBST(WINDRES) +AC_CHECK_TOOL(WINDRES, windres) + # tweak ARFLAGS only if the user didn't set it on the command line AC_SUBST([ARFLAGS]) if test -z "$ARFLAGS" @@ -7108,10 +7112,33 @@ AC_MSG_RESULT([done]) case $host in *-*-mingw*) LIBS="$LIBS -lversion -lshlwapi -lpathcch" + AC_PROG_AWK + if test "$AWK" = "gawk"; then + awk_extra_flag="--non-decimal-data" + fi + AC_MSG_CHECKING([FIELD3]) + FIELD3=$($AWK $awk_extra_flag '\ + /^#define PY_RELEASE_LEVEL_/ {levels[$2]=$3} \ + /^#define PY_MICRO_VERSION[[:space:]]+/ {micro=$3} \ + /^#define PY_RELEASE_LEVEL[[:space:]]+/ {level=levels[$3]} \ + /^#define PY_RELEASE_SERIAL[[:space:]]+/ {serial=$3} \ + END {print micro * 1000 + level * 10 + serial}' \ + $srcdir/Include/patchlevel.h + ) + + AC_MSG_RESULT([${FIELD3}]) + RCFLAGS="$RCFLAGS -DFIELD3=$FIELD3 -O COFF" + + case $host in + i686*) RCFLAGS="$RCFLAGS --target=pe-i386" ;; + x86_64*) RCFLAGS="$RCFLAGS --target=pe-x86-64" ;; + *) ;; + esac ;; *) ;; esac +AC_SUBST(RCFLAGS) # Availability of -O2: AC_CACHE_CHECK([for -O2], [ac_cv_compile_o2], [ From fe710f6c733b8307176ecbccbede142aa0cf6d6a Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:18 +0530 Subject: [PATCH 041/123] fix isselectable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/fileobject.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/fileobject.h b/Include/fileobject.h index 2deef544d667a5..ae4bdac2edcf97 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -30,7 +30,7 @@ Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_UTF8Mode; #endif /* A routine to check if a file descriptor can be select()-ed. */ -#ifdef _MSC_VER +#ifdef MS_WINDOWS /* On Windows, any socket fd can be select()-ed, no matter how high */ #define _PyIsSelectable_fd(FD) (1) #else From f3c6343501ee5ae8afb1eeb4c6eee40cf571cc24 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:23 +0530 Subject: [PATCH 042/123] configure.ac: fix inet_pton check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- configure.ac | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure.ac b/configure.ac index af1f71173bd3bc..87e8fc662c938c 100644 --- a/configure.ac +++ b/configure.ac @@ -5403,10 +5403,14 @@ PY_CHECK_NETDB_FUNC([getprotobyname]) dnl PY_CHECK_SOCKET_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_SOCKET_FUNC], [PY_CHECK_FUNC([$1], [ +#ifdef _WIN32 +#include +#else #include #include #include #include +#endif ])]) PY_CHECK_SOCKET_FUNC([inet_aton]) From 52f1dd30b77b1861d1e7d5449f04d06cd248f605 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:27 +0530 Subject: [PATCH 043/123] pass gen profile ldflags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index d7916604dfc4c2..db921e5a14e1ed 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -898,7 +898,7 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) python_nt.o if test -n "$(DLLLIBRARY)"; then \ $(LDSHARED) -Wl,--out-implib=$@ -o $(DLLLIBRARY) $^ \ - $(LIBS) $(MODLIBS) $(SYSLIBS) python_nt.o; \ + $(LIBS) $(LDFLAGS_NODIST) $(MODLIBS) $(SYSLIBS) python_nt.o; \ else true; \ fi From 49c4354cf120b833700f473cb18537f6d3973eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:29 +0530 Subject: [PATCH 044/123] pkg config windows must link ext with python lib --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 87e8fc662c938c..6843218d1e3f28 100644 --- a/configure.ac +++ b/configure.ac @@ -6284,7 +6284,7 @@ AC_MSG_RESULT([$LDVERSION]) # On Android and Cygwin the shared libraries must be linked with libpython. AC_SUBST([LIBPYTHON]) -if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then +if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin" || test "$MACHDEP" = "win32"); then LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" else LIBPYTHON='' From 2df9f65c69655eb5e557c717eb60e27882dd5671 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:30 +0530 Subject: [PATCH 045/123] importlib bootstrap path sep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Lib/importlib/_bootstrap_external.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 9b8a8dfc5aae28..1fbbc552f28d86 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -42,6 +42,10 @@ path_separators = ['\\', '/'] else: path_separators = ['/'] + +if 'MSYSTEM' in _os.environ: + path_separators = path_separators[::-1] + # Assumption made in _path_join() assert all(len(sep) == 1 for sep in path_separators) path_sep = path_separators[0] From 6c285f5e9b99f99ce9c821f34f38800b36fc1312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:33 +0530 Subject: [PATCH 046/123] warnings fixes --- Modules/_winapi.c | 2 +- Modules/posixmodule.c | 2 +- Modules/socketmodule.h | 2 ++ PC/python_exe.rc | 2 +- PC/pythonw_exe.rc | 2 +- PC/winreg.c | 1 + 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index b5d14686d23c97..abb32f0bd8c718 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1127,7 +1127,7 @@ getattributelist(PyObject *obj, const char *name, AttributeList *attribute_list) DWORD err; BOOL result; PyObject *value; - Py_ssize_t handle_list_size; + Py_ssize_t handle_list_size = 0; DWORD attribute_count = 0; SIZE_T attribute_list_size = 0; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 51b3e525403039..b326a142b51e1d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6280,7 +6280,7 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, /*[clinic end generated code: output=cfcac69d027b82cf input=2fbd62a2f228f8f4]*/ { #ifdef MS_WINDOWS - HANDLE hFile; + HANDLE hFile = 0; FILETIME atime, mtime; #else int result; diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index f5ca00450ee92a..1e92327d04f145 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -68,8 +68,10 @@ struct SOCKADDR_BTH_REDEF { */ # ifdef SIO_GET_MULTICAST_FILTER # include /* for SIO_RCVALL */ +#ifndef __MINGW32__ /* resolve by configure */ # define HAVE_ADDRINFO # define HAVE_SOCKADDR_STORAGE +#endif # define HAVE_GETADDRINFO # define HAVE_GETNAMEINFO # define ENABLE_IPV6 diff --git a/PC/python_exe.rc b/PC/python_exe.rc index c3d3bff019895e..dde0e5384201fa 100644 --- a/PC/python_exe.rc +++ b/PC/python_exe.rc @@ -12,7 +12,7 @@ // current versions of Windows. 1 RT_MANIFEST "python.manifest" -1 ICON DISCARDABLE "icons\python.ico" +1 ICON DISCARDABLE "icons/python.ico" ///////////////////////////////////////////////////////////////////////////// diff --git a/PC/pythonw_exe.rc b/PC/pythonw_exe.rc index 38570b74fa3e02..7ce1043298eac4 100644 --- a/PC/pythonw_exe.rc +++ b/PC/pythonw_exe.rc @@ -12,7 +12,7 @@ // current versions of Windows. 1 RT_MANIFEST "python.manifest" -1 ICON DISCARDABLE "icons\pythonw.ico" +1 ICON DISCARDABLE "icons/pythonw.ico" ///////////////////////////////////////////////////////////////////////////// diff --git a/PC/winreg.c b/PC/winreg.c index 368291d9dafa4e..33ec8ffa07dc99 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -831,6 +831,7 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ) case REG_BINARY: /* ALSO handle ALL unknown data types here. Even if we can't support it natively, we should handle the bits. */ + /* fallthrough */ default: if (retDataSize == 0) { obData = Py_NewRef(Py_None); From 5eaa5ec6b5f6615c85c57eccfbdfda0ed13e97f7 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sat, 17 Jun 2023 00:21:25 +0530 Subject: [PATCH 047/123] fix build testinternalcapi --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 6843218d1e3f28..6ac579985fac8f 100644 --- a/configure.ac +++ b/configure.ac @@ -7912,7 +7912,7 @@ PY_STDLIB_MOD([_overlapped], [test "$MACHDEP" = "win32"], [], [], dnl test modules PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) -PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes], [], [-DPY3_DLLNAME="\"$DLLLIBRARY\""], []) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) From 468ecaee879e02d49d3b77a916ffdef7374cd37d Mon Sep 17 00:00:00 2001 From: jeremyd2019 <4524874+jeremyd2019@users.noreply.github.com> Date: Thu, 17 Jun 2021 18:52:36 +0530 Subject: [PATCH 048/123] clang arm64 Co-authored-by: Naveen M K --- Python/getcompiler.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 3e31c80bdf4fb1..7e7be712a6fbdd 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -7,7 +7,7 @@ // Note the __clang__ conditional has to come before the __GNUC__ one because // clang pretends to be GCC. -#if defined(__clang__) +#if defined(__clang__) && !defined(_WIN32) #define COMPILER "[Clang " __clang_version__ "]" #elif defined(__GNUC__) /* To not break compatibility with things that determine @@ -18,6 +18,8 @@ #define COMP_SEP " " #if defined(__x86_64__) #define ARCH_SUFFIX " 64 bit (AMD64)" +#elif defined(__aarch64__) +#define ARCH_SUFFIX " 64 bit (ARM64)" #else #define ARCH_SUFFIX " 32 bit" #endif @@ -25,7 +27,14 @@ #define COMP_SEP "\n" #define ARCH_SUFFIX "" #endif +#if defined(__clang__) +#define str(x) #x +#define xstr(x) str(x) +#define COMPILER COMP_SEP "[GCC Clang " xstr(__clang_major__) "." \ + xstr(__clang_minor__) "." xstr(__clang_patchlevel__) ARCH_SUFFIX "]" +#else #define COMPILER COMP_SEP "[GCC " __VERSION__ ARCH_SUFFIX "]" +#endif // Generic fallbacks. #elif defined(__cplusplus) #define COMPILER "[C++]" From 1d05f487a3984d84bcdc48bc46b522cfdffe8524 Mon Sep 17 00:00:00 2001 From: jeremyd2019 <4524874+jeremyd2019@users.noreply.github.com> Date: Thu, 17 Jun 2021 18:52:37 +0530 Subject: [PATCH 049/123] configure.ac: set MINGW stack reserve --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 6ac579985fac8f..18d5f415ca4e7e 100644 --- a/configure.ac +++ b/configure.ac @@ -3746,6 +3746,11 @@ then VxWorks*) LINKFORSHARED='-Wl,-export-dynamic';; esac + case $host in + *-*-mingw*) + # for https://bugs.python.org/issue40458 on MINGW + LINKFORSHARED="-Wl,--stack,4194304";; + esac fi AC_MSG_RESULT([$LINKFORSHARED]) From d14cee311242bede8246a22aa5d4fafc02b3441c Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 18 Jun 2021 12:38:11 +0530 Subject: [PATCH 050/123] Don't use os.pathsep to find EOF not all distributions in win32 have them as \ instead check using sys.platform Signed-off-by: Naveen M K --- Lib/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index fadfccf9f77377..938b0a24959cf2 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -419,7 +419,7 @@ def setquit(): The repr of each object contains a hint at how it works. """ - if os.sep == '\\': + if sys.platform == 'win32': eof = 'Ctrl-Z plus Return' else: eof = 'Ctrl-D (i.e. EOF)' From 231b2d4d3802ef14a69564c523892c8b6f96e0b7 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 23 Jun 2021 18:12:12 +0530 Subject: [PATCH 051/123] Fix extension suffix for c-extensions on mingw Python is compiled with various compilers which previously had same platform tags or extension suffix. This can be error prone while loading c-extensions, so now each compiler or runtime has a different extension suffix. Also, changed all extension to end with .pyd rather than .dll file. Fixes https://github.com/msys2/MINGW-packages/issues/8843 Signed-off-by: Naveen M K --- Makefile.pre.in | 6 ++-- Python/dynload_win.c | 6 ---- configure.ac | 77 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index db921e5a14e1ed..cfe863457083ef 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -174,6 +174,7 @@ CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) # Symbols used for using shared libraries SHLIB_SUFFIX= @SHLIB_SUFFIX@ EXT_SUFFIX= @EXT_SUFFIX@ +PYD_PLATFORM_TAG = @PYD_PLATFORM_TAG@ LDSHARED= @LDSHARED@ $(PY_LDFLAGS) BLDSHARED= @BLDSHARED@ $(PY_CORE_LDFLAGS) LDCXXSHARED= @LDCXXSHARED@ $(PY_LDFLAGS) @@ -1403,8 +1404,7 @@ Python/dynload_hpux.o: $(srcdir)/Python/dynload_hpux.c Makefile Python/dynload_win.o: $(srcdir)/Python/dynload_win.c Makefile $(CC) -c $(PY_CORE_CFLAGS) \ - -DSHLIB_SUFFIX='"$(SHLIB_SUFFIX)"' \ - -DEXT_SUFFIX='"$(EXT_SUFFIX)"' \ + -DPYD_PLATFORM_TAG='"$(PYD_PLATFORM_TAG)"' \ -o $@ $(srcdir)/Python/dynload_win.c Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h @@ -2483,7 +2483,7 @@ libainstall: all scripts @if test "$(STATIC_LIBPYTHON)" = 1; then \ if test -d $(LIBRARY); then :; else \ if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ - if test "$(SHLIB_SUFFIX)" = .dll; then \ + if test "$(SHLIB_SUFFIX)" = .dll -o "$(SHLIB_SUFFIX)" = .pyd; then \ $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBPL) ; \ else \ $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \ diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 4bb90ba198b887..26d6036ba79966 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -29,12 +29,6 @@ #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" const char *_PyImport_DynLoadFiletab[] = { -#ifdef EXT_SUFFIX - EXT_SUFFIX, /* include SOABI flags where is encoded debug */ -#endif -#ifdef SHLIB_SUFFIX - "-abi" PYTHON_ABI_STRING SHLIB_SUFFIX, -#endif PYD_TAGGED_SUFFIX, PYD_UNTAGGED_SUFFIX, NULL diff --git a/configure.ac b/configure.ac index 18d5f415ca4e7e..19b866aae8fda1 100644 --- a/configure.ac +++ b/configure.ac @@ -3488,7 +3488,7 @@ if test -z "$SHLIB_SUFFIX"; then *) SHLIB_SUFFIX=.so;; esac case $host_os in - mingw*) SHLIB_SUFFIX=.dll;; + mingw*) SHLIB_SUFFIX=.pyd;; esac fi AC_MSG_RESULT([$SHLIB_SUFFIX]) @@ -6249,6 +6249,68 @@ esac # check for endianness AC_C_BIGENDIAN +AC_SUBST(PYD_PLATFORM_TAG) +# Special case of PYD_PLATFORM_TAG with python build with mingw. +# Python can with compiled with clang or gcc and linked +# to msvcrt or ucrt. To avoid conflicts between them +# we are selecting the extension as based on the compiler +# and the runtime they link to +# gcc + x86_64 + msvcrt = cp{version number}-x86_64 +# gcc + i686 + msvcrt = cp{version number}-i686 +# gcc + x86_64 + ucrt = cp{version number}-x86_64-ucrt +# clang + x86_64 + ucrt = cp{version number}-x86_64-clang +# clang + i686 + ucrt = cp{version number}-i686-clang + +PYD_PLATFORM_TAG="" +case $host in + *-*-mingw*) + # check if we are linking to ucrt + AC_MSG_CHECKING(whether linking to ucrt) + AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ + #include + #ifndef _UCRT + #error no ucrt + #endif + int main(){ return 0; } + ]])],[linking_to_ucrt=yes],[linking_to_ucrt=no]) + AC_MSG_RESULT($linking_to_ucrt) + ;; +esac +case $host_os in + mingw*) + AC_MSG_CHECKING(PYD_PLATFORM_TAG) + case $host in + i686-*-mingw*) + if test -n "${cc_is_clang}"; then + # it is CLANG32 + PYD_PLATFORM_TAG="mingw_i686_clang" + else + if test $linking_to_ucrt = no; then + PYD_PLATFORM_TAG="mingw_i686" + else + PYD_PLATFORM_TAG="mingw_i686_ucrt" + fi + fi + ;; + x86_64-*-mingw*) + if test -n "${cc_is_clang}"; then + # it is CLANG64 + PYD_PLATFORM_TAG="mingw_x86_64_clang" + else + if test $linking_to_ucrt = no; then + PYD_PLATFORM_TAG="mingw_x86_64" + else + PYD_PLATFORM_TAG="mingw_x86_64_ucrt" + fi + fi + ;; + aarch64-*-mingw*) + PYD_PLATFORM_TAG+="mingw_aarch64" + ;; + esac + AC_MSG_RESULT($PYD_PLATFORM_TAG) +esac + # ABI version string for Python extension modules. This appears between the # periods in shared library file names, e.g. foo..so. It is calculated # from the following attributes which affect the ABI of this Python build (in @@ -6281,7 +6343,12 @@ if test "$Py_DEBUG" = 'true' -a "$with_trace_refs" != "yes"; then fi AC_SUBST([EXT_SUFFIX]) -EXT_SUFFIX=.${SOABI}${SHLIB_SUFFIX} +VERSION_NO_DOTS=$(echo $LDVERSION | tr -d .) +if test -n "${PYD_PLATFORM_TAG}"; then + EXT_SUFFIX=".cp${VERSION_NO_DOTS}-${PYD_PLATFORM_TAG}${SHLIB_SUFFIX}" +else + EXT_SUFFIX=.${SOABI}${SHLIB_SUFFIX} +fi AC_MSG_CHECKING([LDVERSION]) LDVERSION='$(VERSION)$(ABIFLAGS)' @@ -7040,12 +7107,6 @@ case "$ac_cv_computed_gotos" in yes*) AC_DEFINE([HAVE_COMPUTED_GOTOS], [1], [Define if the C compiler supports computed gotos.]) esac -case $host_os in - mingw*) - dnl Synchronized with _PyImport_DynLoadFiletab (dynload_win.c) - dnl Do not use more then one dot on this platform ! - EXT_SUFFIX=-$SOABI$SHLIB_SUFFIX;; -esac case $ac_sys_system in AIX*) From 097c217e47efac4e96d52134b9fd42225eaac0b6 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 21 Sep 2021 21:36:58 +0200 Subject: [PATCH 052/123] Change the `get_platform()` method in sysconfig This would possibly fix building wheels when mingw python is used and would be unique to each python same as EXT_SUFFIX. Also, this modifies the `sys.version` argument to include UCRT in it. Signed-off-by: Naveen M K --- Lib/sysconfig.py | 16 ++++++++++++++-- Python/getcompiler.c | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index a0e07b93be6bf6..2a67b846e81b5b 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -813,8 +813,20 @@ def get_platform(): """ if os.name == 'nt': - if 'GCC' in sys.version: - return 'mingw' + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' if 'amd64' in sys.version.lower(): return 'win-amd64' if '(arm)' in sys.version.lower(): diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 7e7be712a6fbdd..275f79997b4241 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -33,8 +33,12 @@ #define COMPILER COMP_SEP "[GCC Clang " xstr(__clang_major__) "." \ xstr(__clang_minor__) "." xstr(__clang_patchlevel__) ARCH_SUFFIX "]" #else +#if defined(_UCRT) +#define COMPILER COMP_SEP "[GCC UCRT " __VERSION__ ARCH_SUFFIX "]" +#else #define COMPILER COMP_SEP "[GCC " __VERSION__ ARCH_SUFFIX "]" #endif +#endif // Generic fallbacks. #elif defined(__cplusplus) #define COMPILER "[C++]" From b8a2d27b7636444187a5df1d8062de7364d2be07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:09 +0530 Subject: [PATCH 053/123] build: Fix ncursesw include lookup Mirror what is already done for libffi; Look it up via pkg-config and use the exported path via sysconfig in setup.py --- Makefile.pre.in | 4 ++++ configure.ac | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index cfe863457083ef..e7c7b89b845597 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -347,6 +347,10 @@ IO_OBJS= \ Modules/_io/bytesio.o \ Modules/_io/stringio.o +########################################################################## + +NCURSESW_INCLUDEDIR= @NCURSESW_INCLUDEDIR@ + ########################################################################## # Parser diff --git a/configure.ac b/configure.ac index 19b866aae8fda1..a566bba8b0b6e9 100644 --- a/configure.ac +++ b/configure.ac @@ -6900,10 +6900,17 @@ AS_VAR_IF([have_panel], [no], [ AC_MSG_RESULT([$have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)]) ]) +if test -n "$PKG_CONFIG"; then + NCURSESW_INCLUDEDIR="`"$PKG_CONFIG" ncursesw --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ *$//'`" +else + NCURSESW_INCLUDEDIR="" +fi +AC_SUBST(NCURSESW_INCLUDEDIR) + # first curses header check ac_save_cppflags="$CPPFLAGS" if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I/usr/include/ncursesw" + CPPFLAGS="$CPPFLAGS -I$NCURSESW_INCLUDEDIR" fi # On Solaris, term.h requires curses.h From 2e26b3e95ea8042448838d9dac14313b0e7e4263 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jul 2021 08:51:32 +0200 Subject: [PATCH 054/123] tests: fix test_bytes %p has different casing with mingw-w64, but it's implementation defined. Change the test to the mingw-w64 variant. --- Lib/test/test_bytes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index b27d43695eef77..f18d38c6dc0e02 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1168,7 +1168,7 @@ def test_from_format(self): if os.name == 'nt': # Windows (MSCRT) - ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) + ptr_format = '0x%0{}x'.format(2 * sizeof_ptr) def ptr_formatter(ptr): return (ptr_format % ptr) else: From 03671ce0bf4312d8cc606046cab628a33f31fc4d Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jul 2021 08:52:50 +0200 Subject: [PATCH 055/123] time: fix strftime not raising for invalid year values It's crt specific and not compiler specific. This fixes a test case in test_time --- Modules/timemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 59078263571f7e..4946cdc2f6563a 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -905,7 +905,7 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } -#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) +#if defined(MS_WINDOWS) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); From 8d8a7d89d49b5ce6b7b7e88698407d062bff8b17 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jul 2021 18:23:58 +0200 Subject: [PATCH 056/123] ctypes: find_library('c') should return None with ucrt Just like with MSVC. This fixes a test in test_ctypes. --- Lib/ctypes/util.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index c550883e7c7d4b..a5a2a3f5dc2d9d 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -31,6 +31,12 @@ def _get_build_version(): # else we don't know what version of the compiler this is return None + def find_msvcrt_mingw(): + is_ucrt = 'clang' in sys.version.lower() or 'ucrt' in sys.version.lower() + if is_ucrt: + return None + return 'msvcrt.dll' + def find_msvcrt(): """Return the name of the VC runtime dll""" version = _get_build_version() @@ -54,6 +60,9 @@ def find_msvcrt(): def find_library(name): if name in ('c', 'm'): + import sysconfig + if sysconfig.get_platform().startswith('mingw'): + return find_msvcrt_mingw() return find_msvcrt() # See MSDN for the REAL search order. for directory in os.environ['PATH'].split(os.pathsep): From fcde121343d904bbaf8744864dafb4d066547b9d Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 25 Jul 2021 13:54:12 +0200 Subject: [PATCH 057/123] build: Disable checks for dlopen/dlfcn While it is (potentially) available with mingw we don't want to use it, so skip any checks for it. --- configure.ac | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index a566bba8b0b6e9..d3b8dbfec2556a 100644 --- a/configure.ac +++ b/configure.ac @@ -3058,7 +3058,7 @@ AC_DEFINE([STDC_HEADERS], [1], # checks for header files AC_CHECK_HEADERS([ \ - alloca.h asm/types.h bluetooth.h conio.h crypt.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ + alloca.h asm/types.h bluetooth.h conio.h crypt.h direct.h endian.h errno.h fcntl.h grp.h \ ieeefp.h io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ linux/random.h linux/soundcard.h \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pty.h \ @@ -3070,6 +3070,13 @@ AC_CHECK_HEADERS([ \ sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h sys/xattr.h sysexits.h syslog.h \ termios.h util.h utime.h utmp.h \ ]) + +case $host in + *-*-mingw*) ;; + *) AC_CHECK_HEADERS([dlfcn.h]);; +esac + + AC_HEADER_DIRENT AC_HEADER_MAJOR @@ -3815,7 +3822,12 @@ AC_SUBST([PERF_TRAMPOLINE_OBJ]) # checks for libraries AC_CHECK_LIB([sendfile], [sendfile]) -AC_CHECK_LIB([dl], [dlopen]) # Dynamic linking for SunOS/Solaris and SYSV + +case $host in + *-*-mingw*) ;; + *) AC_CHECK_LIB([dl], [dlopen]) ;; # Dynamic linking for SunOS/Solaris and SYSV +esac + AC_CHECK_LIB([dld], [shl_load]) # Dynamic linking for HP-UX @@ -6159,7 +6171,10 @@ AS_VAR_IF([ac_cv_broken_sem_getvalue], [yes], [ ) ]) -AC_CHECK_DECLS([RTLD_LAZY, RTLD_NOW, RTLD_GLOBAL, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_DEEPBIND, RTLD_MEMBER], [], [], [[@%:@include ]]) +case $host in + *-*-mingw*) ;; + *) AC_CHECK_DECLS([RTLD_LAZY, RTLD_NOW, RTLD_GLOBAL, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_DEEPBIND, RTLD_MEMBER], [], [], [[@%:@include ]]) +esac # determine what size digit to use for Python's longs AC_MSG_CHECKING([digit size for Python's longs]) From c9f994502e23f5ba2bf8d626a4d84d66ea791f07 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 25 Jul 2021 15:46:01 +0200 Subject: [PATCH 058/123] Fix install location of the import library --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index e7c7b89b845597..fc1ea7d02b0a80 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2488,7 +2488,7 @@ libainstall: all scripts if test -d $(LIBRARY); then :; else \ if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ if test "$(SHLIB_SUFFIX)" = .dll -o "$(SHLIB_SUFFIX)" = .pyd; then \ - $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBPL) ; \ + $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBDIR) ; \ else \ $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \ fi; \ From 5146f248c1292fabf77a9926359acd963454a0eb Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 25 Jul 2021 16:53:35 +0200 Subject: [PATCH 059/123] build: Integrate venvlauncher build/installation into the Makefile This is required for venv creation on Windows. Ideally this would use the venv specific launcher (PC/launcher.c), but a copy of main binary seems to work as well for now. --- Makefile.pre.in | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index fc1ea7d02b0a80..073a770591f5e0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -170,6 +170,7 @@ BINLIBDEST= @BINLIBDEST@ LIBDEST= $(SCRIPTDIR)/python$(VERSION) INCLUDEPY= $(INCLUDEDIR)/python$(LDVERSION) CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) +VENVLAUNCHERDIR= $(BINLIBDEST)/venv/scripts/nt # Symbols used for using shared libraries SHLIB_SUFFIX= @SHLIB_SUFFIX@ @@ -290,6 +291,8 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) BUILDPYTHONW= pythonw$(BUILDEXE) +BUILDVENVLAUNCHER= venvlauncher$(BUILDEXE) +BUILDVENVWLAUNCHER= venvwlauncher$(BUILDEXE) HOSTRUNNER= @HOSTRUNNER@ @@ -630,7 +633,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: all .PHONY: build_all -build_all: check-clean-src $(BUILDPYTHON) $(BUILDPYTHONW) platform sharedmods \ +build_all: check-clean-src $(BUILDPYTHON) $(BUILDPYTHONW) $(BUILDVENVLAUNCHER) $(BUILDVENVWLAUNCHER) platform sharedmods \ gdbhooks Programs/_testembed scripts checksharedmods rundsymutil .PHONY: build_wasm @@ -813,6 +816,14 @@ $(BUILDPYTHONW): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) pythonw $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) python_exe.o $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) python_exe.o +# FIXME: build these from PC/launcher.c instead +$(BUILDVENVLAUNCHER): $(BUILDPYTHON) + cp $(BUILDPYTHON) $(BUILDVENVLAUNCHER) + +# FIXME: build these from PC/launcher.c instead +$(BUILDVENVWLAUNCHER): $(BUILDPYTHONW) + cp $(BUILDPYTHONW) $(BUILDVENVWLAUNCHER) + platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -1988,7 +1999,7 @@ sharedinstall: all # This goes into $(exec_prefix) .PHONY: altbininstall altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ - @for i in $(BINDIR) $(LIBDIR); \ + @for i in $(BINDIR) $(LIBDIR) $(VENVLAUNCHERDIR); \ do \ if test ! -d $(DESTDIR)$$i; then \ echo "Creating directory $$i"; \ @@ -1999,6 +2010,8 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ if test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(BINDIR)/python3w$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(VENVLAUNCHERDIR)/python$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(VENVLAUNCHERDIR)/pythonw$(EXE); \ else \ $(INSTALL_PROGRAM) $(STRIPFLAG) Mac/pythonw $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ fi From 2a336b4478ea1fb69962c6d507a5b636facc458e Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 1 Aug 2021 15:18:50 +0200 Subject: [PATCH 060/123] configure.ac: set _WIN32_WINNT version --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index d3b8dbfec2556a..d6e86040bb7ee7 100644 --- a/configure.ac +++ b/configure.ac @@ -4597,6 +4597,11 @@ done IFS=$as_save_IFS AC_MSG_RESULT([$DBM_CFLAGS $DBM_LIBS]) +case $host in + *-*-mingw*) + CFLAGS_NODIST="$CFLAGS_NODIST -D_WIN32_WINNT=0x0602";; +esac + # Determine if windows modules should be used. AC_SUBST(USE_WIN32_MODULE) USE_WIN32_MODULE='#' From 2ceeedcaf13090ec01313f2c2e221877bd043226 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 1 Aug 2021 18:35:17 +0200 Subject: [PATCH 061/123] configure.ac: don't check for clock_ functions They shouldn't be exposed on Windows and lead to winpthread being linked in --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index d6e86040bb7ee7..e8844317f794e5 100644 --- a/configure.ac +++ b/configure.ac @@ -5540,6 +5540,9 @@ WITH_SAVE_ENV([ ]) ]) +case $host in + *-*-mingw*) ;; + *) AC_CHECK_FUNCS([clock_gettime], [], [ AC_CHECK_LIB([rt], [clock_gettime], [ LIBS="$LIBS -lrt" @@ -5560,6 +5563,8 @@ AC_CHECK_FUNCS([clock_settime], [], [ AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) ]) ]) + ;; +esac AC_CHECK_FUNCS([clock_nanosleep], [], [ AC_CHECK_LIB([rt], [clock_nanosleep], [ From 1cb7dcd873211b8aa5600648b7935f915b0f32a0 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 8 Aug 2021 10:17:35 +0200 Subject: [PATCH 062/123] expanduser: normpath paths coming from env vars This makes sure we get the same paths as with related functions in pathlib. --- Lib/ntpath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index c4855ae11f36f0..f1d33337e41e6f 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -380,7 +380,7 @@ def expanduser(path): if 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] elif not 'HOMEPATH' in os.environ: - return path + return os.path.normpath(path) else: try: drive = os.environ['HOMEDRIVE'] @@ -407,7 +407,7 @@ def expanduser(path): if isinstance(path, bytes): userhome = os.fsencode(userhome) - return userhome + path[i:] + return os.path.normpath(userhome) + path[i:] # Expand paths containing shell variable substitutions. From 23ec3fc831776324010c6efe0c79583fa9a2ada7 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 18 Jun 2021 17:51:59 +0530 Subject: [PATCH 063/123] CI: test the build and add some mingw specific tests - Use actions/setup-python for setting up correct version in cross build - CI: add cross llvm-mingw jobs - CI: fix sed pattern for python-config.py shebang The shebang (`#!`) does not include `${pkgdir}` so this sed did nothing - CI: remove --without-c-locale-coercion No longer needed since the default now works on Windows - Make sure we always use the stdlib distutils - CI: update actions and images ubuntu-18.04 is gone now - CI: updates for mstorsjo/llvm-mingw moving to 20.04 - CI: 3.10 -> 3.11 - CI: remove `--with-system-libmpdec` for cross builds - CI: move gcc cross build to Ubuntu for newer mingw-w64 The build requires headers that are only available in v12 (processsnapshot.h) and Arch is still on v11. So move things to Ubuntu. Similarely to other cross builds remove the system libs configure flags, those were never there and now fail properly with the 3.12 build system. Co-authored-by: Christoph Reiter --- .github/workflows/mingw.yml | 289 +++++++++++++++++++++++++++++++++ mingw_ignorefile.txt | 34 ++++ mingw_smoketests.py | 313 ++++++++++++++++++++++++++++++++++++ 3 files changed, 636 insertions(+) create mode 100644 .github/workflows/mingw.yml create mode 100644 mingw_ignorefile.txt create mode 100644 mingw_smoketests.py diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml new file mode 100644 index 00000000000000..2b606fb992dd0e --- /dev/null +++ b/.github/workflows/mingw.yml @@ -0,0 +1,289 @@ +name: Build +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + msystem: ['MINGW64','MINGW32','UCRT64','CLANG64'] + include: + - msystem: MINGW64 + prefix: mingw-w64-x86_64 + - msystem: MINGW32 + prefix: mingw-w64-i686 + - msystem: UCRT64 + prefix: mingw-w64-ucrt-x86_64 + - msystem: CLANG64 + prefix: mingw-w64-clang-x86_64 + steps: + - name: Setup git + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - uses: actions/checkout@v3 + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + release: false + update: true + install: >- + make + binutils + autoconf + autoconf-archive + automake-wrapper + tar + gzip + ${{ matrix.prefix }}-toolchain + ${{ matrix.prefix }}-expat + ${{ matrix.prefix }}-bzip2 + ${{ matrix.prefix }}-libffi + ${{ matrix.prefix }}-mpdecimal + ${{ matrix.prefix }}-ncurses + ${{ matrix.prefix }}-openssl + ${{ matrix.prefix }}-sqlite3 + ${{ matrix.prefix }}-tcl + ${{ matrix.prefix }}-tk + ${{ matrix.prefix }}-zlib + ${{ matrix.prefix }}-xz + ${{ matrix.prefix }}-tzdata + + - name: Build Python + shell: msys2 {0} + run: | + set -ex + + if [[ "${{ matrix.msystem }}" == CLANG* ]]; then + export CC=clang + export CXX=clang++ + fi + autoreconf -vfi + + rm -Rf _build && mkdir _build && cd _build + + ../configure \ + --prefix=${MINGW_PREFIX} \ + --host=${MINGW_CHOST} \ + --build=${MINGW_CHOST} \ + --enable-shared \ + --with-system-expat \ + --with-system-libmpdec \ + --without-ensurepip \ + --enable-loadable-sqlite-extensions \ + --with-tzpath=${MINGW_PREFIX}/share/zoneinfo \ + --enable-optimizations + + make -j8 + + - name: Run Smoke Test (build) + shell: msys2 {0} + run: | + SMOKETESTS="$(pwd)/mingw_smoketests.py" + cd _build + ./python.exe "$SMOKETESTS" + MSYSTEM= ./python.exe "$SMOKETESTS" + + - name: Run tests + shell: msys2 {0} + run: | + IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + cd _build + MSYSTEM= ./python.exe -m test -j8 --ignorefile "$IGNOREFILE" -W + + - name: Run broken tests + continue-on-error: true + shell: msys2 {0} + run: | + IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + cd _build + MSYSTEM= ./python.exe -m test -j8 --matchfile "$IGNOREFILE" -W + + - name: Install + shell: msys2 {0} + run: | + set -ex + cd _build + + pkgdir=python_pkgdir + + make -j1 install DESTDIR="${pkgdir}" + + # Fix shebangs + _pybasever=$(./python.exe -c "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}');") + for fscripts in 2to3 2to3-${_pybasever} idle3 idle${_pybasever} pydoc3 pydoc${_pybasever}; do + sed -i "s|$(cygpath -w ${MINGW_PREFIX} | sed 's|\\|\\\\|g')/bin/python${_pybasever}.exe|/usr/bin/env python${_pybasever}.exe|g" "${pkgdir}${MINGW_PREFIX}"/bin/${fscripts} + done + sed -i "s|#!${MINGW_PREFIX}/bin/python${_pybasever}.exe|#!/usr/bin/env python${_pybasever}.exe|" "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${_pybasever}/python-config.py + + # Create version-less aliases + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3.exe "${pkgdir}${MINGW_PREFIX}"/bin/python.exe + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3w.exe "${pkgdir}${MINGW_PREFIX}"/bin/pythonw.exe + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3-config "${pkgdir}${MINGW_PREFIX}"/bin/python-config + cp "${pkgdir}${MINGW_PREFIX}"/bin/idle3 "${pkgdir}${MINGW_PREFIX}"/bin/idle + cp "${pkgdir}${MINGW_PREFIX}"/bin/pydoc3 "${pkgdir}${MINGW_PREFIX}"/bin/pydoc + + - name: Run Smoke Test (installed) + shell: msys2 {0} + run: | + export PYTHONTZPATH="${MINGW_PREFIX}/share/zoneinfo" + SMOKETESTS="$(pwd)/mingw_smoketests.py" + cd _build + cd python_pkgdir/${MINGW_PREFIX}/bin + ./python.exe "$SMOKETESTS" + MSYSTEM= ./python.exe "$SMOKETESTS" + + - name: Compress + if: always() + shell: msys2 {0} + run: | + cd _build + tar -zcf python.tar.gz python_pkgdir/ + + - name: Upload + uses: actions/upload-artifact@v3 + if: always() + with: + name: build-${{ matrix.msystem }} + path: _build/python.tar.gz + + cross-gcc-x86_64: + runs-on: ubuntu-latest + container: + image: ubuntu:24.10 + steps: + - uses: actions/checkout@v3 + - name: Install deps + run: | + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + gcc-mingw-w64-x86-64 \ + g++-mingw-w64-x86-64 \ + autoconf-archive \ + autoconf \ + automake \ + zip \ + make \ + pkg-config + + - uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Check Python Version + run: | + which python + python --version + + - name: Build + run: | + autoreconf -vfi + + mkdir _build && cd _build + + ../configure \ + --host=x86_64-w64-mingw32 \ + --build=x86_64-pc-linux-gnu \ + --enable-shared \ + --without-ensurepip \ + --enable-loadable-sqlite-extensions \ + --with-build-python=yes + + make -j8 + + make install DESTDIR="$(pwd)/install" + + - name: 'Zip files' + run: | + zip -r install.zip _build/install + + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: build-cross-gcc-x86_64 + path: install.zip + + cross-gcc-x86_64-test: + needs: [cross-gcc-x86_64] + runs-on: windows-latest + steps: + - uses: actions/download-artifact@v3 + with: + name: build-cross-gcc-x86_64 + + - name: 'Run tests' + run: | + 7z x install.zip + ./_build/install/usr/local/bin/python3.exe -c "import sysconfig, pprint; pprint.pprint(sysconfig.get_config_vars())" + + + cross-llvm-mingw: + runs-on: ubuntu-latest + container: + image: mstorsjo/llvm-mingw:latest + strategy: + fail-fast: false + matrix: + arch: ['x86_64', 'i686', 'aarch64', 'armv7'] + steps: + - uses: actions/checkout@v3 + + - name: Install deps + run: | + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -qqy software-properties-common + add-apt-repository --yes ppa:deadsnakes/ppa + apt-get update -qq + apt-get install -qqy autoconf-archive python3.12-dev python3.12 + + - name: Build + run: | + autoreconf -vfi + + mkdir _build && cd _build + + export CC=${{ matrix.arch }}-w64-mingw32-clang + export CXX=${CC}++ + ../configure \ + --host=${{ matrix.arch }}-w64-mingw32 \ + --build=x86_64-pc-linux-gnu \ + --enable-shared \ + --without-ensurepip \ + --without-c-locale-coercion \ + --enable-loadable-sqlite-extensions \ + --with-build-python=yes + + make -j8 + + make install DESTDIR="$(pwd)/install" + + - name: 'Zip files' + run: | + zip -r install.zip _build/install + + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: build-cross-llvm-mingw-${{ matrix.arch }} + path: install.zip + + cross-llvm-mingw-test: + needs: [cross-llvm-mingw] + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + arch: ['x86_64', 'i686'] + steps: + - uses: actions/download-artifact@v3 + with: + name: build-cross-llvm-mingw-${{ matrix.arch }} + + - name: 'Run tests' + run: | + 7z x install.zip + ./_build/install/usr/local/bin/python3.exe -c "import sysconfig, pprint; pprint.pprint(sysconfig.get_config_vars())" + + diff --git a/mingw_ignorefile.txt b/mingw_ignorefile.txt new file mode 100644 index 00000000000000..dc3802e6c51313 --- /dev/null +++ b/mingw_ignorefile.txt @@ -0,0 +1,34 @@ +ctypes.test.test_loading.LoaderTest.test_load_dll_with_flags +distutils.tests.test_bdist_dumb.BuildDumbTestCase.test_simple_built +distutils.tests.test_cygwinccompiler.CygwinCCompilerTestCase.test_get_versions +distutils.tests.test_util.UtilTestCase.test_change_root +test.datetimetester.TestLocalTimeDisambiguation_Fast.* +test.datetimetester.TestLocalTimeDisambiguation_Pure.* +test.test_cmath.CMathTests.test_specific_values +test.test_cmd_line_script.CmdLineTest.test_consistent_sys_path_for_direct_execution +test.test_compileall.CommandLineTestsNoSourceEpoch.* +test.test_compileall.CommandLineTestsWithSourceEpoch.* +test.test_compileall.CompileallTestsWithoutSourceEpoch.* +test.test_compileall.CompileallTestsWithSourceEpoch.* +test.test_import.ImportTests.test_dll_dependency_import +test.test_math.MathTests.* +test.test_ntpath.NtCommonTest.test_import +test.test_os.StatAttributeTests.test_stat_block_device +test.test_os.TestScandir.test_attributes +test.test_os.UtimeTests.test_large_time +test.test_platform.PlatformTest.test_architecture_via_symlink +test.test_regrtest.ProgramsTestCase.test_pcbuild_rt +test.test_regrtest.ProgramsTestCase.test_tools_buildbot_test +test.test_site._pthFileTests.* +test.test_site.HelperFunctionsTests.* +test.test_site.StartupImportTests.* +test.test_ssl.* +test.test_strptime.CalculationTests.* +test.test_strptime.StrptimeTests.test_weekday +test.test_strptime.TimeRETests.test_compile +test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date +test.test_venv.BasicTest.* +test.test_venv.EnsurePipTest.* +# flaky +test.test__xxsubinterpreters.* +test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe \ No newline at end of file diff --git a/mingw_smoketests.py b/mingw_smoketests.py new file mode 100644 index 00000000000000..d92c74ac248053 --- /dev/null +++ b/mingw_smoketests.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# Copyright 2017 Christoph Reiter +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""The goal of this test suite is collect tests for update regressions +and to test msys2 related modifications like for path handling. +Feel free to extend. +""" + +import os +import unittest +import sysconfig + +if "MSYSTEM" in os.environ: + SEP = "/" +else: + SEP = "\\" + +_UCRT = "clang" in sysconfig.get_platform() or "ucrt" in sysconfig.get_platform() + + +class Tests(unittest.TestCase): + + def test_zoneinfo(self): + # https://github.com/msys2-contrib/cpython-mingw/issues/32 + import zoneinfo + self.assertTrue(any(os.path.exists(p) for p in zoneinfo.TZPATH)) + zoneinfo.ZoneInfo("America/Sao_Paulo") + + def test_userdir_path_sep(self): + # Make sure os.path and pathlib use the same path separators + from unittest import mock + from os.path import expanduser + from pathlib import Path + + profiles = ["C:\\foo", "C:/foo"] + for profile in profiles: + with mock.patch.dict(os.environ, {"USERPROFILE": profile}): + self.assertEqual(expanduser("~"), os.path.normpath(expanduser("~"))) + self.assertEqual(str(Path("~").expanduser()), expanduser("~")) + self.assertEqual(str(Path.home()), expanduser("~")) + + def test_sysconfig_schemes(self): + # https://github.com/msys2/MINGW-packages/issues/9319 + import sysconfig + from distutils.dist import Distribution + from distutils.command.install import install + + names = ['scripts', 'purelib', 'platlib', 'data', 'include'] + for scheme in ["nt", "nt_user"]: + for name in names: + c = install(Distribution({"name": "foobar"})) + c.user = (scheme == "nt_user") + c.finalize_options() + if name == "include": + dist_path = os.path.dirname(getattr(c, "install_" + "headers")) + else: + dist_path = getattr(c, "install_" + name) + sys_path = sysconfig.get_path(name, scheme) + self.assertEqual(dist_path, sys_path, (scheme, name)) + + def test_ctypes_find_library(self): + from ctypes.util import find_library + from ctypes import cdll + self.assertTrue(cdll.msvcrt) + if _UCRT: + self.assertIsNone(find_library('c')) + else: + self.assertEqual(find_library('c'), 'msvcrt.dll') + + def test_ctypes_dlopen(self): + import ctypes + import sys + self.assertEqual(ctypes.RTLD_GLOBAL, 0) + self.assertEqual(ctypes.RTLD_GLOBAL, ctypes.RTLD_LOCAL) + self.assertFalse(hasattr(sys, 'setdlopenflags')) + self.assertFalse(hasattr(sys, 'getdlopenflags')) + self.assertFalse([n for n in dir(os) if n.startswith("RTLD_")]) + + def test_time_no_unix_stuff(self): + import time + self.assertFalse([n for n in dir(time) if n.startswith("clock_")]) + self.assertFalse([n for n in dir(time) if n.startswith("CLOCK_")]) + self.assertFalse([n for n in dir(time) if n.startswith("pthread_")]) + self.assertFalse(hasattr(time, 'tzset')) + + def test_strftime(self): + import time + with self.assertRaises(ValueError): + time.strftime('%Y', (12345,) + (0,) * 8) + + def test_sep(self): + self.assertEqual(os.sep, SEP) + + def test_module_file_path(self): + import asyncio + import zlib + self.assertEqual(zlib.__file__, os.path.normpath(zlib.__file__)) + self.assertEqual(asyncio.__file__, os.path.normpath(asyncio.__file__)) + + def test_importlib_frozen_path_sep(self): + import importlib._bootstrap_external + self.assertEqual(importlib._bootstrap_external.path_sep, SEP) + + def test_os_commonpath(self): + self.assertEqual( + os.path.commonpath( + [os.path.join("C:", os.sep, "foo", "bar"), + os.path.join("C:", os.sep, "foo")]), + os.path.join("C:", os.sep, "foo")) + + def test_pathlib(self): + import pathlib + + p = pathlib.Path("foo") / pathlib.Path("foo") + self.assertEqual(str(p), os.path.normpath(p)) + + def test_modules_import(self): + import sqlite3 + import ssl + import ctypes + import curses + + def test_c_modules_import(self): + import _decimal + + def test_socket_inet_ntop(self): + import socket + self.assertTrue(hasattr(socket, "inet_ntop")) + + def test_socket_inet_pton(self): + import socket + self.assertTrue(hasattr(socket, "inet_pton")) + + def test_multiprocessing_queue(self): + from multiprocessing import Queue + Queue(0) + + #def test_socket_timout_normal_error(self): + # import urllib.request + # from urllib.error import URLError + + # try: + # urllib.request.urlopen( + # 'http://localhost', timeout=0.0001).close() + # except URLError: + # pass + + def test_threads(self): + from concurrent.futures import ThreadPoolExecutor + + with ThreadPoolExecutor(1) as pool: + for res in pool.map(lambda *x: None, range(10000)): + pass + + def test_sysconfig(self): + import sysconfig + # This should be able to execute without exceptions + sysconfig.get_config_vars() + + def test_sqlite_enable_load_extension(self): + # Make sure --enable-loadable-sqlite-extensions is used + import sqlite3 + self.assertTrue(sqlite3.Connection.enable_load_extension) + + def test_venv_creation(self): + import tempfile + import venv + import subprocess + import shutil + with tempfile.TemporaryDirectory() as tmp: + builder = venv.EnvBuilder() + builder.create(tmp) + assert os.path.exists(os.path.join(tmp, "bin", "activate")) + assert os.path.exists(os.path.join(tmp, "bin", "python.exe")) + assert os.path.exists(os.path.join(tmp, "bin", "python3.exe")) + subprocess.check_call([shutil.which("bash.exe"), os.path.join(tmp, "bin", "activate")]) + + def test_has_mktime(self): + from time import mktime, gmtime + mktime(gmtime()) + + def test_platform_things(self): + import sys + import sysconfig + import platform + import importlib.machinery + self.assertEqual(sys.implementation.name, "cpython") + self.assertEqual(sys.platform, "win32") + self.assertTrue(sysconfig.get_platform().startswith("mingw")) + self.assertTrue(sysconfig.get_config_var('SOABI').startswith("cpython-")) + ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') + self.assertTrue(ext_suffix.endswith(".pyd")) + self.assertTrue("mingw" in ext_suffix) + self.assertEqual(sysconfig.get_config_var('SHLIB_SUFFIX'), ".pyd") + ext_suffixes = importlib.machinery.EXTENSION_SUFFIXES + self.assertTrue(ext_suffix in ext_suffixes) + self.assertTrue(".pyd" in ext_suffixes) + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2]))) + self.assertEqual(platform.python_implementation(), "CPython") + self.assertEqual(platform.system(), "Windows") + self.assertTrue(isinstance(sys.api_version, int) and sys.api_version > 0) + + def test_c_ext_build(self): + import tempfile + import sys + import subprocess + import textwrap + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmppro: + subprocess.check_call([sys.executable, "-m", "ensurepip", "--user"]) + with Path(tmppro, "setup.py").open("w") as f: + f.write( + textwrap.dedent( + """\ + from setuptools import setup, Extension + + setup( + name='cwrapper', + version='1.0', + ext_modules=[ + Extension( + 'cwrapper', + sources=['cwrapper.c']), + ], + ) + """ + ) + ) + with Path(tmppro, "cwrapper.c").open("w") as f: + f.write( + textwrap.dedent( + """\ + #include + static PyObject * + helloworld(PyObject *self, PyObject *args) + { + printf("Hello World\\n"); + Py_RETURN_NONE; + } + static PyMethodDef + myMethods[] = { + { "helloworld", helloworld, METH_NOARGS, "Prints Hello World" }, + { NULL, NULL, 0, NULL } + }; + static struct PyModuleDef cwrapper = { + PyModuleDef_HEAD_INIT, + "cwrapper", + "Test Module", + -1, + myMethods + }; + + PyMODINIT_FUNC + PyInit_cwrapper(void) + { + return PyModule_Create(&cwrapper); + } + """ + ) + ) + subprocess.check_call( + [sys.executable, "-c", "import struct"], + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + "wheel", + ], + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + tmppro, + ], + ) + subprocess.check_call( + [sys.executable, "-c", "import cwrapper"], + ) + + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From 42eaab00bb6de358d97ef0e23d6a7a7f1960ed17 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 12 Oct 2021 18:35:21 +0530 Subject: [PATCH 064/123] Define PY3_DLLNAME to fix build PY3_DLLNAME is used in Python/pathconfig.c but isn't defined. I guess it is useless but it will fix the build. Co-authored-by: jeremyd2019 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e8844317f794e5..2b37fe3f5bd1a8 100644 --- a/configure.ac +++ b/configure.ac @@ -5143,7 +5143,7 @@ then *-*-mingw*) DYNLOADFILE="dynload_win.o" extra_machdep_objs="$extra_machdep_objs PC/dl_nt.o" - CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"'" + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"' -DPY3_DLLNAME='L\"$DLLLIBRARY\"'" ;; esac fi From 41c2be507eb793fd5703bb2123f05f7e1fb81937 Mon Sep 17 00:00:00 2001 From: Long Nguyen Date: Sun, 14 Nov 2021 14:55:35 +0700 Subject: [PATCH 065/123] _testconsole.c: Fix casing & path sep --- PC/_testconsole.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PC/_testconsole.c b/PC/_testconsole.c index 3221b985d01ba0..47fbe6feb33832 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -116,7 +116,7 @@ _testconsole_read_output_impl(PyObject *module, PyObject *file) Py_RETURN_NONE; } -#include "clinic\_testconsole.c.h" +#include "clinic/_testconsole.c.h" PyMethodDef testconsole_methods[] = { _TESTCONSOLE_WRITE_INPUT_METHODDEF From 2c9a95669e3c4d1b44ea81c1d4d77f2459c0749e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Fri, 19 Nov 2021 12:18:06 +0200 Subject: [PATCH 066/123] Return consistent architecture markers for python on mingw/armv7 --- Lib/sysconfig.py | 2 ++ Python/getcompiler.c | 2 ++ configure.ac | 3 +++ 3 files changed, 7 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 2a67b846e81b5b..0150c18f452565 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -823,6 +823,8 @@ def get_platform(): return 'mingw_x86_64_clang' if 'arm64' in sys.version.lower(): return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' return 'mingw_i686_clang' if 'amd64' in sys.version.lower(): return 'mingw_x86_64' diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 275f79997b4241..4b0b9b328ba2f8 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -20,6 +20,8 @@ #define ARCH_SUFFIX " 64 bit (AMD64)" #elif defined(__aarch64__) #define ARCH_SUFFIX " 64 bit (ARM64)" +#elif defined(__arm__) +#define ARCH_SUFFIX " 32 bit (ARM)" #else #define ARCH_SUFFIX " 32 bit" #endif diff --git a/configure.ac b/configure.ac index 2b37fe3f5bd1a8..4c757d872edbf0 100644 --- a/configure.ac +++ b/configure.ac @@ -6332,6 +6332,9 @@ case $host_os in aarch64-*-mingw*) PYD_PLATFORM_TAG+="mingw_aarch64" ;; + armv7-*-mingw*) + PYD_PLATFORM_TAG+="mingw_armv7" + ;; esac AC_MSG_RESULT($PYD_PLATFORM_TAG) esac From c45f5f0f98a0af66d059417b0cc37652eedf6f37 Mon Sep 17 00:00:00 2001 From: jeremyd2019 Date: Mon, 22 Nov 2021 16:04:41 -0800 Subject: [PATCH 067/123] handle ncursesw pkg-config when cross-compiling strip extra args from pkg-config --cflags-only-I, setup.py is only expecting a single path. Use pkg-config ncursesw include dir even if cross-compiling (PKG_CONFIG_PATH should be set in that case). --- configure.ac | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 4c757d872edbf0..ec3962e1f67d1f 100644 --- a/configure.ac +++ b/configure.ac @@ -6929,7 +6929,7 @@ AS_VAR_IF([have_panel], [no], [ ]) if test -n "$PKG_CONFIG"; then - NCURSESW_INCLUDEDIR="`"$PKG_CONFIG" ncursesw --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ *$//'`" + NCURSESW_INCLUDEDIR="`"$PKG_CONFIG" ncursesw --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ .*$//'`" else NCURSESW_INCLUDEDIR="" fi @@ -6937,9 +6937,7 @@ AC_SUBST(NCURSESW_INCLUDEDIR) # first curses header check ac_save_cppflags="$CPPFLAGS" -if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I$NCURSESW_INCLUDEDIR" -fi +CPPFLAGS="$CPPFLAGS -I$NCURSESW_INCLUDEDIR" # On Solaris, term.h requires curses.h AC_CHECK_HEADERS([term.h], [], [], [ From 76fa04a3505741b459e64d006d95635366c84ac3 Mon Sep 17 00:00:00 2001 From: jeremyd2019 Date: Sat, 22 Jan 2022 11:33:00 -0800 Subject: [PATCH 068/123] mingw_smoketests: fix _UCRT condition The prior detection missed ARM (32 and 64) being UCRT. Since there are fewer non-UCRT platforms, list those two instead of trying to extend the list of UCRT. --- mingw_smoketests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index d92c74ac248053..ce95846fa2914f 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -34,7 +34,7 @@ else: SEP = "\\" -_UCRT = "clang" in sysconfig.get_platform() or "ucrt" in sysconfig.get_platform() +_UCRT = sysconfig.get_platform() not in ('mingw_x86_64', 'mingw_i686') class Tests(unittest.TestCase): From 3debb2dd3a029dda9f2cbdfc84b581ffb787156e Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 19 Jan 2022 19:39:20 +0530 Subject: [PATCH 069/123] Modify `sys.winver` to match upstream With this change `sys.winver` will add the Arch for which python was compiled on, for example in 32-bits, `sys.winver` will be `3.10-32`, for arm32 it would be `3.10-arm32` and so on. See https://github.com/msys2-contrib/cpython-mingw/issues/40 --- configure.ac | 22 ++++++++++++++++++---- mingw_smoketests.py | 9 ++++++++- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index ec3962e1f67d1f..e8d63cde226334 100644 --- a/configure.ac +++ b/configure.ac @@ -5141,10 +5141,24 @@ then esac case $host in *-*-mingw*) - DYNLOADFILE="dynload_win.o" - extra_machdep_objs="$extra_machdep_objs PC/dl_nt.o" - CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"' -DPY3_DLLNAME='L\"$DLLLIBRARY\"'" - ;; + DYNLOADFILE="dynload_win.o" + extra_machdep_objs="$extra_machdep_objs PC/dl_nt.o" + CFLAGS_NODIST="$CFLAGS_NODIST -DPY3_DLLNAME='L\"$DLLLIBRARY\"'" + case $host in + i686*) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"${VERSION}-32\"'" + ;; + armv7*) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"${VERSION}-arm32\"'" + ;; + aarch64*) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"${VERSION}-arm64\"'" + ;; + *) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"'" + ;; + esac + ;; esac fi AC_MSG_RESULT([$DYNLOADFILE]) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index ce95846fa2914f..aa766592062dab 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -214,7 +214,14 @@ def test_platform_things(self): ext_suffixes = importlib.machinery.EXTENSION_SUFFIXES self.assertTrue(ext_suffix in ext_suffixes) self.assertTrue(".pyd" in ext_suffixes) - self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2]))) + if sysconfig.get_platform().startswith('mingw_i686'): + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2])) + '-32') + elif sysconfig.get_platform().startswith('mingw_aarch64'): + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2])) + '-arm64') + elif sysconfig.get_platform().startswith('mingw_armv7'): + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2])) + '-arm32') + else: + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2]))) self.assertEqual(platform.python_implementation(), "CPython") self.assertEqual(platform.system(), "Windows") self.assertTrue(isinstance(sys.api_version, int) and sys.api_version > 0) From 4283ac7a8903e95d944f58b750292bf842ba3a25 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 19 Jan 2022 20:01:45 +0530 Subject: [PATCH 070/123] Change user site-packages path to include the environment info This should avoid mixing of user site-packages between python from various environments. Previously, the user site-packages should be located at `~/.local/lib/python3.10` for all environment including 32-bits variants which caused problems with 64-bit trying to load 32-bit extensions. Now this path will be changed to `~/.local/lib/python3.10-`, for example, in CLANG64 this would be `~/.local/lib/python3.10-mingw_x86_64_clang`. Fixes https://github.com/msys2-contrib/cpython-mingw/issues/40 --- Lib/site.py | 28 +++++++++++++++++++++++++--- Lib/sysconfig.py | 26 +++++++++++++++----------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 938b0a24959cf2..c1ecd6ec4739c3 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -310,14 +310,36 @@ def joinuser(*args): return joinuser("~", ".local") +# Copy of sysconfig.get_platform() but only for MinGW +def _get_platform(): + if os.name == 'nt': + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' + return sys.platform # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info - if os.name == 'nt' and not _POSIX_BUILD: - ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\Python{ver_nodot}\\site-packages' + if os.name == 'nt': + if not _POSIX_BUILD: + ver_nodot = sys.winver.replace('.', '') + return f'{userbase}\\Python{ver_nodot}\\site-packages' + return f'{userbase}/lib/python{version[0]}.{version[1]}-{_get_platform()}/site-packages' if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/python/site-packages' diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 0150c18f452565..916c1e82b2d9b7 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -141,20 +141,20 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/lib/python{py_version_short}', - 'platstdlib': '{userbase}/lib/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/python{py_version_short_plat}', + 'platstdlib': '{userbase}/lib/python{py_version_short_plat}', + 'purelib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'include': '{userbase}/include/python{py_version_short_plat}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/python{py_version_short_plat}', + 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short_plat}', + 'purelib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'include': '{userbase}/include/python{py_version_short_plat}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -702,6 +702,10 @@ def _init_config_vars(): _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: _CONFIG_VARS['py_version_nodot_plat'] = '' + if os.name == 'nt' and _POSIX_BUILD: + _CONFIG_VARS['py_version_short_plat'] = f'{_PY_VERSION_SHORT}-{get_platform()}' + else: + _CONFIG_VARS['py_version_short_plat'] = _PY_VERSION_SHORT if os.name == 'nt' and not _POSIX_BUILD: _init_non_posix(_CONFIG_VARS) @@ -789,7 +793,7 @@ def get_config_var(name): """ return get_config_vars().get(name) - +# make sure to change site._get_platform() while changing this function def get_platform(): """Return a string that identifies the current platform. From 5410b83fa92587f8dea118ecadc15da92540307c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Thu, 21 Apr 2022 14:25:51 +0300 Subject: [PATCH 071/123] configure: Include a header in the check for _beginthread Previously, the test tried compiling a call to the _beginthread function without either declaring the function (and its parameters) or including the corresponding header. Since Clang 15 (which still is under development, so this may still change before it's released) [1], implicit function declarations are a hard error by default, when building code in C99 mode (or newer). [1] https://github.com/llvm/llvm-project/commit/7d644e1215b376ec5e915df9ea2eeb56e2d94626 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e8d63cde226334..8a0f79c35c90b4 100644 --- a/configure.ac +++ b/configure.ac @@ -2870,7 +2870,7 @@ AC_MSG_RESULT([$with_nt_threads]) if test $with_nt_threads = yes ; then AC_MSG_CHECKING([whether linking with nt-threads work]) AC_LINK_IFELSE([ - AC_LANG_PROGRAM([[]],[[_beginthread(0, 0, 0);]]) + AC_LANG_PROGRAM([[#include ]],[[_beginthread(0, 0, 0);]]) ], [AC_MSG_RESULT([yes])], [AC_MSG_ERROR([failed to link with nt-threads])]) From c7f7a151a52a08e67af88939d38502b284c7b8ca Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 22 Apr 2022 17:31:19 +0200 Subject: [PATCH 072/123] configure.ac: Default to --without-c-locale-coercion on Windows --with-c-locale-coercion otherwise defaults to yes and enables code that isn't compatible on Windows, mainly because the feature is Unix related. Default to "no" on Windows instead. Fixes #36 --- configure.ac | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 8a0f79c35c90b4..2258bb50c77589 100644 --- a/configure.ac +++ b/configure.ac @@ -5030,11 +5030,14 @@ AC_MSG_RESULT([$with_freelists]) AC_MSG_CHECKING([for --with-c-locale-coercion]) AC_ARG_WITH( [c-locale-coercion], - [AS_HELP_STRING([--with-c-locale-coercion], [enable C locale coercion to a UTF-8 based locale (default is yes)])]) + [AS_HELP_STRING([--with-c-locale-coercion], [enable C locale coercion to a UTF-8 based locale (default is yes on Unix, no on Windows)])]) if test -z "$with_c_locale_coercion" then - with_c_locale_coercion="yes" + case $host in + *-*-mingw*) with_c_locale_coercion="no";; + *) with_c_locale_coercion="yes";; + esac fi if test "$with_c_locale_coercion" != "no" then From 3b22a18b72c74fd89851837ed8910442e1b62f41 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 5 Jun 2022 20:28:53 +0530 Subject: [PATCH 073/123] Fix failing tests - In `test_sysconfig`, ignore `test_sysconfig.TestSysConfig.test_user_similar` test failure. - Copy `get_platform()` from from distutils.utils to test_importlib/test_windows.py. - In `test_tcl`, ignore `test_tcl.TclTest.testLoadWithUNC` test failure. - Disable `test.test_asynchat.TestAsynchat.test_line_terminator2`, seems flaky. - skip some more flaky tests - some basic fixes for test_getpath - test_sysconfig.py: fix tests related to mingw --- Lib/test/test_getpath.py | 3 +++ Lib/test/test_importlib/test_windows.py | 17 +++++++++++++++++ Lib/test/test_sysconfig.py | 8 ++++++-- mingw_ignorefile.txt | 13 +++++++++++-- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 3b613a569b981a..4dc2d305730d67 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -837,6 +837,7 @@ def test_symlink_buildpath_macos(self): ENV_PYTHONHOME="", ENV_PYTHONEXECUTABLE="", ENV___PYVENV_LAUNCHER__="", + ENV_MSYSTEM="", argv0="", py_setpath="", real_executable="", @@ -876,6 +877,7 @@ def __init__(self, *a, argv0=None, config=None, **kw): self.update(DEFAULT_NAMESPACE) self["config"] = DEFAULT_CONFIG.copy() self["os_name"] = "nt" + self["is_mingw"] = 0 self["PLATLIBDIR"] = "DLLs" self["PYWINVER"] = "9.8-XY" self["VPATH"] = r"..\.." @@ -1052,6 +1054,7 @@ def __init__(self, *a, argv0=None, config=None, **kw): self.update(DEFAULT_NAMESPACE) self["config"] = DEFAULT_CONFIG.copy() self["os_name"] = "posix" + self["is_mingw"] = 0 self["PLATLIBDIR"] = "lib" self["WITH_NEXT_FRAMEWORK"] = 0 super().__init__(*a, **kw) diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index 40b8aa1787fe08..03b5a98974f3fa 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -24,6 +24,23 @@ def get_platform(): 'x64' : 'win-amd64', 'arm' : 'win-arm32', } + if os.name == 'nt': + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' if ('VSCMD_ARG_TGT_ARCH' in os.environ and os.environ['VSCMD_ARG_TGT_ARCH'] in TARGET_TO_PLAT): return TARGET_TO_PLAT[os.environ['VSCMD_ARG_TGT_ARCH']] diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 67647e1b787ce7..a883037c1199d2 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -19,7 +19,7 @@ from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes, _main) + _expand_vars, _get_preferred_schemes, _main, _POSIX_BUILD) import _osx_support @@ -197,7 +197,7 @@ def test_nt_venv_scheme(self): self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): - if sys.platform == 'win32': + if not _POSIX_BUILD and sys.platform == 'win32': self.assertEqual( sysconfig.get_path('scripts', scheme='venv'), sysconfig.get_path('scripts', scheme='nt_venv') @@ -388,6 +388,10 @@ def test_user_similar(self): if HAS_USER_BASE: user_path = get_path(name, 'posix_user') expected = os.path.normpath(global_path.replace(base, user, 1)) + if os.name == 'nt' and _POSIX_BUILD: + expected = expected.replace( + f'python{sysconfig.get_python_version()}', + f'python{sysconfig.get_python_version()}-{get_platform()}') # bpo-44860: platlib of posix_user doesn't use sys.platlibdir, # whereas posix_prefix does. if name == 'platlib': diff --git a/mingw_ignorefile.txt b/mingw_ignorefile.txt index dc3802e6c51313..e692d7ff26eb4d 100644 --- a/mingw_ignorefile.txt +++ b/mingw_ignorefile.txt @@ -1,4 +1,4 @@ -ctypes.test.test_loading.LoaderTest.test_load_dll_with_flags +test.test_ctypes.test_loading.LoaderTest.test_load_dll_with_flags distutils.tests.test_bdist_dumb.BuildDumbTestCase.test_simple_built distutils.tests.test_cygwinccompiler.CygwinCCompilerTestCase.test_get_versions distutils.tests.test_util.UtilTestCase.test_change_root @@ -29,6 +29,15 @@ test.test_strptime.TimeRETests.test_compile test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date test.test_venv.BasicTest.* test.test_venv.EnsurePipTest.* +test.test_sysconfig.TestSysConfig.test_user_similar +test.test_tcl.TclTest.testLoadWithUNC +test.test_wmi # flaky test.test__xxsubinterpreters.* -test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe \ No newline at end of file +test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe +test.test_asynchat.TestAsynchat.test_line_terminator2 +test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_gc_aclose_09 +test.test_concurrent_futures.ThreadPoolShutdownTest.test_interpreter_shutdown +test.test_asynchat.TestNotConnected.test_disallow_negative_terminator +test.test_logging.SysLogHandlerTest.* +test.test_logging.IPv6SysLogHandlerTest.* From b31b52e48af3922d36429a53c390699a113ecb60 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 22 Sep 2022 17:51:33 +0200 Subject: [PATCH 074/123] Don't change os.sep with an empty MSYSTEM env var, not just a missing one Up until now this didn't really happen when calling from cygwin because empty env vars were removed before Python would run. But https://github.com/msys2/msys2-runtime/pull/101 changed that. To avoid breaking users that did something like MSYSTEM= python ... not only check that MSYSTEM isn't set but also that it isn't empty when deciding if os.sep/os.altsep should be switched. Also, guard the msystem env check to execute only on MINGW Co-authored-by: Naveen M K --- Lib/importlib/_bootstrap_external.py | 2 +- Lib/ntpath.py | 2 +- Python/pathconfig.c | 20 +++++++++++--------- mingw_smoketests.py | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 1fbbc552f28d86..a177fa72cfd1c3 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -43,7 +43,7 @@ else: path_separators = ['/'] -if 'MSYSTEM' in _os.environ: +if _os.environ.get('MSYSTEM', ''): path_separators = path_separators[::-1] # Assumption made in _path_join() diff --git a/Lib/ntpath.py b/Lib/ntpath.py index f1d33337e41e6f..1323ac23b2bcba 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -21,7 +21,7 @@ import genericpath from genericpath import * -if sys.platform == "win32" and "MSYSTEM" in os.environ: +if sys.platform == "win32" and os.environ.get("MSYSTEM", ""): sep = '/' altsep = '\\' else: diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 3d684d68ff12df..1195a04fe11437 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -2,7 +2,7 @@ #include "Python.h" #include "marshal.h" // PyMarshal_ReadObjectFromString -#include "osdefs.h" // DELIM +#include "osdefs.h" // DELIM, SEP #include "pycore_initconfig.h" #include "pycore_fileutils.h" #include "pycore_pathconfig.h" @@ -50,7 +50,6 @@ Py_StartsWithW(const wchar_t * str, const wchar_t * prefix) char Py_GetSepA(const char *name) { - char* msystem = (char*)2; /* So that non Windows use / as sep */ static char sep = '\0'; #ifdef _WIN32 /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx @@ -65,12 +64,14 @@ Py_GetSepA(const char *name) if (sep != '\0') return sep; #if defined(__MINGW32__) - msystem = Py_GETENV("MSYSTEM"); -#endif - if (msystem != NULL) + char* msystem = getenv("MSYSTEM"); + if (msystem != NULL && strcmp(msystem, "") != 0) sep = '/'; else sep = '\\'; +#else + sep = SEP; +#endif return sep; } @@ -103,7 +104,6 @@ Py_NormalizeSepsA(char *name) wchar_t Py_GetSepW(const wchar_t *name) { - char* msystem = (char*)2; /* So that non Windows use / as sep */ static wchar_t sep = L'\0'; #ifdef _WIN32 /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx @@ -118,12 +118,14 @@ Py_GetSepW(const wchar_t *name) if (sep != L'\0') return sep; #if defined(__MINGW32__) - msystem = Py_GETENV("MSYSTEM"); -#endif - if (msystem != NULL) + char* msystem = getenv("MSYSTEM"); + if (msystem != NULL && strcmp(msystem, "") != 0) sep = L'/'; else sep = L'\\'; +#else + sep = SEP; +#endif return sep; } diff --git a/mingw_smoketests.py b/mingw_smoketests.py index aa766592062dab..2a8bd160b0042a 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -29,7 +29,7 @@ import unittest import sysconfig -if "MSYSTEM" in os.environ: +if os.environ.get("MSYSTEM", ""): SEP = "/" else: SEP = "\\" From df24bb7652e592633ec854a102a2656e6960726f Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 13:12:56 +0530 Subject: [PATCH 075/123] def VPATH when compiling `Python/sysmodule.c` --- Makefile.pre.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 073a770591f5e0..a8f358763f8b28 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1425,6 +1425,7 @@ Python/dynload_win.o: $(srcdir)/Python/dynload_win.c Makefile Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h $(CC) -c $(PY_CORE_CFLAGS) \ -DABIFLAGS='"$(ABIFLAGS)"' \ + -DVPATH='"$(VPATH)"' \ $(MULTIARCH_CPPFLAGS) \ -o $@ $(srcdir)/Python/sysmodule.c From 418477991a2db99c1c99db0e0c101eff825f3c10 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 13:17:48 +0530 Subject: [PATCH 076/123] Make `_Py_CheckPython3` extern it's declared in headers but defined as static here, remove it also run `_Py_CheckPython3` only when using MSVC --- Python/dynload_win.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 26d6036ba79966..1f90e21a77fecc 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -171,8 +171,7 @@ static char *GetPythonImport (HINSTANCE hModule) Return whether the DLL was found. */ extern HMODULE PyWin_DLLhModule; -static int -_Py_CheckPython3(void) +int _Py_CheckPython3(void) { static int python3_checked = 0; static HANDLE hPython3; @@ -226,9 +225,9 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, dl_funcptr p; char funcname[258], *import_python; -#ifdef Py_ENABLE_SHARED - _Py_CheckPython3(); -#endif /* Py_ENABLE_SHARED */ +#ifdef _MSC_VER + _Py_CheckPython3(); +#endif /* _MSC_VER */ wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL); if (wpathname == NULL) From 67b6568629dda03665d459be98b5f626bafdacd3 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 14:20:44 +0530 Subject: [PATCH 077/123] link with bcrypt --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 2258bb50c77589..01344fea13fc78 100644 --- a/configure.ac +++ b/configure.ac @@ -7231,7 +7231,7 @@ AC_MSG_RESULT([done]) # For mingw build need additional library for linking case $host in *-*-mingw*) - LIBS="$LIBS -lversion -lshlwapi -lpathcch" + LIBS="$LIBS -lversion -lshlwapi -lpathcch -lbcrypt" AC_PROG_AWK if test "$AWK" = "gawk"; then awk_extra_flag="--non-decimal-data" From 2726b17d9ed1909839d5295a7efea173a9cd47f7 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 14:29:50 +0530 Subject: [PATCH 078/123] correctly find native python it's required for regen-modules --- configure.ac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 01344fea13fc78..56bd3c1895036d 100644 --- a/configure.ac +++ b/configure.ac @@ -204,9 +204,11 @@ AC_SUBST([FREEZE_MODULE]) AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) +NATIVE_PYTHON_SEARCH_PATH_MINGW=`echo $host | grep -Eq 'mingw*' && echo "$MINGW_PREFIX/bin" || echo $PATH` AC_CHECK_PROGS([PYTHON_FOR_REGEN], [python$PACKAGE_VERSION python3.12 python3.11 python3.10 python3 python], - [python3]) + [python3], + [$NATIVE_PYTHON_SEARCH_PATH_MINGW]) AC_SUBST([PYTHON_FOR_REGEN]) AC_MSG_CHECKING([Python for regen version]) From fa9ea58652bd42fa2e141dfb8e5dc6806db98066 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 14:30:09 +0530 Subject: [PATCH 079/123] Add extra flags for `_bootstrap_python` --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a8f358763f8b28..595792fb4f29a5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1111,7 +1111,7 @@ BOOTSTRAP_HEADERS = \ Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS) _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local - $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \ + $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ -municode -mwindows $(LIBRARY_OBJS_OMIT_FROZEN) \ Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS) From 9d6027c1cdd7ee18a306fd4d604dcc6e005942fd Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 18:32:58 +0530 Subject: [PATCH 080/123] posixmodule: undefine `HAVE_OPENDIR` opendir is detected by configure on mingw-w64, and for some reason things don't work as expected. For example, os.listdir always returns the cwd's directory listing instead of the one specified. By un-defining, this, os.listdir will use the one which uses native windows API. --- Modules/posixmodule.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b326a142b51e1d..53b93d2e424917 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -373,6 +373,12 @@ corresponding Unix manual entries for more information on calls."); #elif defined(__MINGW32__) /* GCC for windows hosts */ /* getlogin is detected by configure on mingw-w64 */ # undef HAVE_GETLOGIN +/* opendir is detected by configure on mingw-w64, and for some reason +things don't work as expected. For example, os.listdir always returns +the cwd's directory listing instead of the one specified. By +un-defining, this, os.listdir will use the one which uses native +windows API. */ +# undef HAVE_OPENDIR /*# define HAVE_GETCWD 1 - detected by configure*/ # define HAVE_GETPPID 1 # define HAVE_GETLOGIN 1 From dc3cb94a0bc7eed78f457fa6c52322cddafb9171 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 18:43:59 +0530 Subject: [PATCH 081/123] getpath.py: add support for mingw - always normalize the PREFIX to an absolute path - use `/` when MSYSTEM is defined --- Include/pylifecycle.h | 1 + Modules/getpath.c | 9 ++++++++ Modules/getpath.py | 48 +++++++++++++++++++++++++++++++------------ Python/fileutils.c | 41 +++++++++++++++++++++--------------- Python/pathconfig.c | 2 +- 5 files changed, 71 insertions(+), 30 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 50d8b76d74b610..5d292014ff50a7 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -21,6 +21,7 @@ PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); +PyAPI_FUNC(wchar_t) Py_GetAltSepW(const wchar_t *); PyAPI_FUNC(wchar_t) Py_GetSepW(const wchar_t *); PyAPI_FUNC(char) Py_GetSepA(const char *); diff --git a/Modules/getpath.c b/Modules/getpath.c index 0a3100007511b1..de7efb628393e9 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -68,6 +68,7 @@ getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) if (path) { wchar_t *abs; if (_Py_abspath((const wchar_t *)_Py_normpath(path, -1), &abs) == 0 && abs) { + abs = _Py_normpath(abs, -1); r = PyUnicode_FromWideChar(abs, -1); PyMem_RawFree((void *)abs); } else { @@ -881,6 +882,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) #else !decode_to_dict(dict, "os_name", "posix") || #endif +#ifdef __MINGW32__ + !int_to_dict(dict, "is_mingw", 1) || +#else + !int_to_dict(dict, "is_mingw", 0) || +#endif #ifdef WITH_NEXT_FRAMEWORK !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 1) || #else @@ -907,6 +913,9 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !funcs_to_dict(dict, config->pathconfig_warnings) || #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || +#endif +#ifdef __MINGW32__ + !env_to_dict(dict, "ENV_MSYSTEM", 0) || #endif PyDict_SetItemString(dict, "__builtins__", PyEval_GetBuiltins()) < 0 ) { diff --git a/Modules/getpath.py b/Modules/getpath.py index 9913fcba497d30..6867e98eba535f 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -30,6 +30,7 @@ # ** Values known at compile time ** # os_name -- [in] one of 'nt', 'posix', 'darwin' +# is_mingw -- [in] True if targeting MinGW # PREFIX -- [in] sysconfig.get_config_var(...) # EXEC_PREFIX -- [in] sysconfig.get_config_var(...) # PYTHONPATH -- [in] sysconfig.get_config_var(...) @@ -51,6 +52,7 @@ # ENV_PYTHONHOME -- [in] getenv(...) # ENV_PYTHONEXECUTABLE -- [in] getenv(...) # ENV___PYVENV_LAUNCHER__ -- [in] getenv(...) +# ENV_MSYSTEM -- [in] getenv(...) # ** Values calculated at runtime ** # config -- [in/out] dict of the PyConfig structure @@ -185,8 +187,27 @@ ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' DELIM = ':' SEP = '/' + ALTSEP = None -elif os_name == 'nt': +elif os_name == 'nt' and is_mingw: + BUILDDIR_TXT = 'pybuilddir.txt' + BUILD_LANDMARK = 'Modules/Setup.local' + DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] + VENV_LANDMARK = 'pyvenv.cfg' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + DELIM = ';' + if ENV_MSYSTEM: + SEP = '/' + ALTSEP = '\\' + else: + SEP = '\\' + ALTSEP = '/' + +elif os_name == 'nt': # MSVC BUILDDIR_TXT = 'pybuilddir.txt' BUILD_LANDMARK = f'{VPATH}\\Modules\\Setup.local' DEFAULT_PROGRAM_NAME = f'python' @@ -199,6 +220,7 @@ WINREG_KEY = f'SOFTWARE\\Python\\PythonCore\\{PYWINVER}\\PythonPath' DELIM = ';' SEP = '\\' + ALTSEP = '/' # ****************************************************************************** @@ -263,10 +285,10 @@ def search_up(prefix, *landmarks, test=isfile): if not executable: executable = real_executable -if not executable and SEP in program_name: +if not executable and (SEP in program_name or + (ALTSEP and ALTSEP in program_name)): # Resolve partial path program_name against current directory executable = abspath(program_name) - if not executable: # All platforms default to real_executable if known at this # stage. POSIX does not set this value. @@ -497,15 +519,15 @@ def search_up(prefix, *landmarks, test=isfile): except (FileNotFoundError, PermissionError): if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)): build_prefix = joinpath(real_executable_dir, VPATH) - if os_name == 'nt': + if os_name == 'nt' and not is_mingw: # QUIRK: Windows builds need platstdlib_dir to be the executable # dir. Normally the builddir marker handles this, but in this # case we need to correct manually. platstdlib_dir = real_executable_dir if build_prefix: - if os_name == 'nt': - # QUIRK: No searching for more landmarks on Windows + if os_name == 'nt' and not is_mingw: + # QUIRK: No searching for more landmarks on MSVC build_stdlib_prefix = build_prefix else: build_stdlib_prefix = search_up(build_prefix, *BUILDSTDLIB_LANDMARKS) @@ -597,7 +619,7 @@ def search_up(prefix, *landmarks, test=isfile): # Detect exec_prefix by searching from executable for the platstdlib_dir if PLATSTDLIB_LANDMARK and not exec_prefix: - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows always assumed these were the same # gh-100320: Our PYDs are assumed to be relative to the Lib directory # (that is, prefix) rather than the executable (that is, executable_dir) @@ -607,7 +629,7 @@ def search_up(prefix, *landmarks, test=isfile): if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: If DLLs is missing on Windows, don't warn, just assume # that they're in exec_prefix if not platstdlib_dir: @@ -660,7 +682,7 @@ def search_up(prefix, *landmarks, test=isfile): pythonpath.append(abspath(p)) # Then add the default zip file - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows uses the library directory rather than the prefix if library: library_dir = dirname(library) @@ -673,7 +695,7 @@ def search_up(prefix, *landmarks, test=isfile): else: pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) - if os_name == 'nt' and use_environment and winreg: + if (not is_mingw) and os_name == 'nt' and use_environment and winreg: # QUIRK: Windows also lists paths in the registry. Paths are stored # as the default value of each subkey of # {HKCU,HKLM}\Software\Python\PythonCore\{winver}\PythonPath @@ -714,7 +736,7 @@ def search_up(prefix, *landmarks, test=isfile): if not platstdlib_dir and exec_prefix: platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK) - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows generates paths differently if platstdlib_dir: pythonpath.append(platstdlib_dir) @@ -742,8 +764,8 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. -if os_name != 'nt' and build_prefix: - prefix = config.get('prefix') or PREFIX +if (os_name != 'nt' or is_mingw) and build_prefix: + prefix = config.get('prefix') or abspath(PREFIX) exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix diff --git a/Python/fileutils.c b/Python/fileutils.c index 6d3b04ff5bb9e6..50c78f18d779dc 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2189,7 +2189,11 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) } #ifdef MS_WINDOWS - return _PyOS_getfullpathname(path, abspath_p); + if (_PyOS_getfullpathname(path, abspath_p) < 0){ + return -1; + } + *abspath_p = _Py_normpath(*abspath_p, -1); + return 0; #else wchar_t cwd[MAXPATHLEN + 1]; cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; @@ -2401,11 +2405,16 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) wchar_t *minP2 = path; // the beginning of the destination range wchar_t lastC = L'\0'; // the last ljusted character, p2[-1] in most cases + const wchar_t sep = Py_GetSepW(NULL); +#ifdef ALTSEP + const wchar_t altsep = Py_GetAltSepW(NULL); +#endif + #define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) #ifdef ALTSEP -#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP) +#define IS_SEP(x) (*(x) == sep || *(x) == altsep) #else -#define IS_SEP(x) (*(x) == SEP) +#define IS_SEP(x) (*(x) == sep) #endif #define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) @@ -2416,7 +2425,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) path++; } p1 = p2 = minP2 = path; - lastC = SEP; + lastC = sep; } #ifdef MS_WINDOWS // Skip past drive segment and update minP2 @@ -2430,13 +2439,13 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) // and network paths, including the first segment. else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) { int sepCount = 2; - *p2++ = SEP; - *p2++ = SEP; + *p2++ = sep; + *p2++ = sep; p1 += 2; for (; !IS_END(p1) && sepCount; ++p1) { if (IS_SEP(p1)) { --sepCount; - *p2++ = lastC = SEP; + *p2++ = lastC = sep; } else { *p2++ = lastC = *p1; } @@ -2449,7 +2458,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) *p2++ = *p1++; *p2++ = *p1++; minP2 = p2 - 1; // Absolute path has SEP at minP2 - lastC = SEP; + lastC = sep; } #endif /* MS_WINDOWS */ @@ -2457,18 +2466,18 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) for (; !IS_END(p1); ++p1) { wchar_t c = *p1; #ifdef ALTSEP - if (c == ALTSEP) { - c = SEP; + if (c == altsep) { + c = sep; } #endif - if (lastC == SEP) { + if (lastC == sep) { if (c == L'.') { int sep_at_1 = SEP_OR_END(&p1[1]); int sep_at_2 = !sep_at_1 && SEP_OR_END(&p1[2]); if (sep_at_2 && p1[1] == L'.') { wchar_t *p3 = p2; - while (p3 != minP2 && *--p3 == SEP) { } - while (p3 != minP2 && *(p3 - 1) != SEP) { --p3; } + while (p3 != minP2 && *--p3 == sep) { } + while (p3 != minP2 && *(p3 - 1) != sep) { --p3; } if (p2 == minP2 || (p3[0] == L'.' && p3[1] == L'.' && IS_SEP(&p3[2]))) { @@ -2477,7 +2486,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) *p2++ = L'.'; *p2++ = L'.'; lastC = L'.'; - } else if (p3[0] == SEP) { + } else if (p3[0] == sep) { // Absolute path, so absorb segment p2 = p3 + 1; } else { @@ -2488,7 +2497,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) } else { *p2++ = lastC = c; } - } else if (c == SEP) { + } else if (c == sep) { } else { *p2++ = lastC = c; } @@ -2498,7 +2507,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) } *p2 = L'\0'; if (p2 != minP2) { - while (--p2 != minP2 && *p2 == SEP) { + while (--p2 != minP2 && *p2 == sep) { *p2 = L'\0'; } } else { diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 1195a04fe11437..08a4a374ea6023 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -129,7 +129,7 @@ Py_GetSepW(const wchar_t *name) return sep; } -static wchar_t +wchar_t Py_GetAltSepW(const wchar_t *name) { char sep = Py_GetSepW(name); From 3ac5f2fc5c320fd0c185e3faca5f48fdfcd9a3c8 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 20:17:23 +0530 Subject: [PATCH 082/123] Don't build _posixsubprocess on Windows. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 56bd3c1895036d..c1188de2148304 100644 --- a/configure.ac +++ b/configure.ac @@ -7879,7 +7879,6 @@ PY_STDLIB_MOD_SIMPLE([_json]) PY_STDLIB_MOD_SIMPLE([_lsprof]) PY_STDLIB_MOD_SIMPLE([_opcode]) PY_STDLIB_MOD_SIMPLE([_pickle]) -PY_STDLIB_MOD_SIMPLE([_posixsubprocess]) PY_STDLIB_MOD_SIMPLE([_queue]) PY_STDLIB_MOD_SIMPLE([_random]) PY_STDLIB_MOD_SIMPLE([select]) @@ -7932,6 +7931,7 @@ PY_STDLIB_MOD([_scproxy], PY_STDLIB_MOD([spwd], [], [test "$ac_cv_func_getspent" = yes -o "$ac_cv_func_getspnam" = yes]) PY_STDLIB_MOD([syslog], [], [test "$ac_cv_header_syslog_h" = yes]) PY_STDLIB_MOD([termios], [], [test "$ac_cv_header_termios_h" = yes]) +PY_STDLIB_MOD([_posixsubprocess], [], [test "$MACHDEP" != "win32"]) dnl _elementtree loads libexpat via CAPI hook in pyexpat PY_STDLIB_MOD([pyexpat], From abc76648354d0819db3a8f1920816f9b88a4fdb6 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 20:35:27 +0530 Subject: [PATCH 083/123] `_ssl`: link with `ws2_32` --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c1188de2148304..e05eb8f8411813 100644 --- a/configure.ac +++ b/configure.ac @@ -8013,7 +8013,7 @@ PY_STDLIB_MOD([_lzma], [], [test "$have_liblzma" = yes], dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], - [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) + [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS -lws2_32]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) From f1102c49003fc833f2fd95ed966e9cc0f519bc42 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 23:00:43 +0530 Subject: [PATCH 084/123] Always normalize path in abspath --- Lib/ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 1323ac23b2bcba..15830f903b8050 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -594,7 +594,7 @@ def abspath(path): def abspath(path): """Return the absolute version of a path.""" try: - return _getfullpathname(normpath(path)) + return normpath(_getfullpathname(normpath(path))) except (OSError, ValueError): # See gh-75230, handle outside for cleaner traceback pass From a54560597e21804f0bf1c368b32756a98c8833a8 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 21 Jun 2023 15:42:39 +0530 Subject: [PATCH 085/123] Include `winsock.h` when checking for netdb function also move `gethostname` to that check --- configure.ac | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index e05eb8f8411813..5a59971522674a 100644 --- a/configure.ac +++ b/configure.ac @@ -5205,7 +5205,7 @@ AC_CHECK_FUNCS([ \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \ - getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \ + getgrnam_r getgrouplist getgroups getitimer getloadavg getlogin \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \ @@ -5432,7 +5432,13 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [have_liblzma=yes], [ ]) dnl PY_CHECK_NETDB_FUNC(FUNCTION) -AC_DEFUN([PY_CHECK_NETDB_FUNC], [PY_CHECK_FUNC([$1], [@%:@include ])]) +AC_DEFUN([PY_CHECK_NETDB_FUNC], [PY_CHECK_FUNC([$1], [ +#ifdef _WIN32 + #include +#else + #include +#endif +])]) PY_CHECK_NETDB_FUNC([hstrerror]) dnl not available in WASI yet @@ -5441,6 +5447,7 @@ PY_CHECK_NETDB_FUNC([getservbyport]) PY_CHECK_NETDB_FUNC([gethostbyname]) PY_CHECK_NETDB_FUNC([gethostbyaddr]) PY_CHECK_NETDB_FUNC([getprotobyname]) +PY_CHECK_NETDB_FUNC([gethostname]) dnl PY_CHECK_SOCKET_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_SOCKET_FUNC], [PY_CHECK_FUNC([$1], [ From 97bbe1f347c5c11c4ba8ff93081a4ebf423a9aa0 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 23 Jun 2023 20:13:23 +0530 Subject: [PATCH 086/123] include `_multiprocessing/semaphore.c` on win32 while building `_multiprocessing` extension also, always build that module on win32 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 5a59971522674a..8462adb859fa0f 100644 --- a/configure.ac +++ b/configure.ac @@ -7897,7 +7897,7 @@ PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules PY_STDLIB_MOD([_multiprocessing], - [], [test "$ac_cv_func_sem_unlink" = "yes"], + [], [test "$ac_cv_func_sem_unlink" = "yes" -o "$MACHDEP" = "win32"], [-I\$(srcdir)/Modules/_multiprocessing]) PY_STDLIB_MOD([_posixshmem], [], [test "$have_posix_shmem" = "yes"], From c2545fb9dfe545c382bc2affe103929f89ba9176 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 23 Jun 2023 20:15:38 +0530 Subject: [PATCH 087/123] configure: build `mmap` module on win32 --- configure.ac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 8462adb859fa0f..f1947afde4783b 100644 --- a/configure.ac +++ b/configure.ac @@ -7917,7 +7917,9 @@ PY_STDLIB_MOD([fcntl], [], [test "$ac_cv_header_sys_ioctl_h" = "yes" -a "$ac_cv_header_fcntl_h" = "yes"], [], [$FCNTL_LIBS]) PY_STDLIB_MOD([mmap], - [], [test "$ac_cv_header_sys_mman_h" = "yes" -a "$ac_cv_header_sys_stat_h" = "yes"]) + [], m4_flatten([test "$ac_cv_header_sys_mman_h" = "yes" + -a "$ac_cv_header_sys_stat_h" = "yes" + -o "$MACHDEP" = "win32"])) PY_STDLIB_MOD([_socket], [], m4_flatten([test "$ac_cv_header_sys_socket_h" = "yes" -a "$ac_cv_header_sys_types_h" = "yes" From 1441e1caba63fc7bf98d9893f78ee89021441110 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 23 Jun 2023 20:49:15 +0530 Subject: [PATCH 088/123] venv creation fixes --- Lib/venv/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index aeb522c3199ca2..7266f83fef2434 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -13,6 +13,7 @@ import types import shlex +from sysconfig import _POSIX_BUILD CORE_VENV_DEPS = ('pip',) logger = logging.getLogger(__name__) @@ -330,7 +331,7 @@ def setup_python(self, context): if not os.path.islink(path): os.chmod(path, 0o755) else: - if self.symlinks: + if self.symlinks and not _POSIX_BUILD: # For symlinking, we need a complete copy of the root directory # If symlinks fail, you'll get unnecessary copies of files, but # we assume that if you've opted into symlinks on Windows then @@ -354,6 +355,12 @@ def setup_python(self, context): if os.path.lexists(src): copier(src, os.path.join(binpath, suffix)) + if _POSIX_BUILD: + # copy from python/pythonw so the venvlauncher magic in symlink_or_copy triggers + copier(os.path.join(dirname, 'python.exe'), os.path.join(binpath, 'python3.exe')) + copier(os.path.join(dirname, 'python.exe'), os.path.join(binpath, 'python%d.%d.exe' % sys.version_info[:2])) + copier(os.path.join(dirname, 'pythonw.exe'), os.path.join(binpath, 'python3w.exe')) + if sysconfig.is_python_build(): # copy init.tcl for root, dirs, files in os.walk(context.python_dir): @@ -378,6 +385,7 @@ def _call_new_python(self, context, *py_args, **kwargs): env['VIRTUAL_ENV'] = context.env_dir env.pop('PYTHONHOME', None) env.pop('PYTHONPATH', None) + env.pop("MSYSTEM", None) kwargs['cwd'] = context.env_dir kwargs['executable'] = context.env_exec_cmd subprocess.check_output(args, **kwargs) From 117dc5b8a3542a10d5d872d44a3417c69608bf01 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 23 Jun 2023 23:34:01 +0530 Subject: [PATCH 089/123] move the `shutdown` function where `winsock.h` is included --- configure.ac | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f1947afde4783b..305a401dbb8bf2 100644 --- a/configure.ac +++ b/configure.ac @@ -5218,7 +5218,7 @@ AC_CHECK_FUNCS([ \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ setitimer setlocale setpgid setpgrp setpriority setregid setresgid \ - setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ + setresuid setreuid setsid setuid setvbuf sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ @@ -5448,6 +5448,7 @@ PY_CHECK_NETDB_FUNC([gethostbyname]) PY_CHECK_NETDB_FUNC([gethostbyaddr]) PY_CHECK_NETDB_FUNC([getprotobyname]) PY_CHECK_NETDB_FUNC([gethostname]) +PY_CHECK_NETDB_FUNC([shutdown]) dnl PY_CHECK_SOCKET_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_SOCKET_FUNC], [PY_CHECK_FUNC([$1], [ From 0cce81ea6f088db362e06fa354b952ff9f9ff17b Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jun 2023 21:47:32 +0200 Subject: [PATCH 090/123] configure.ac: set BUILDEXEEXT and EXEEXT --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 305a401dbb8bf2..b3c3ab020df37e 100644 --- a/configure.ac +++ b/configure.ac @@ -1409,6 +1409,7 @@ AC_ARG_WITH([suffix], [Emscripten/browser*], [EXEEXT=.js], [Emscripten/node*], [EXEEXT=.js], [WASI/*], [EXEEXT=.wasm], + [MINGW*], [EXEEXT=.exe], [EXEEXT=] ) ]) @@ -1435,6 +1436,10 @@ else fi rmdir CaseSensitiveTestDir +AS_CASE([$ac_sys_system], + [MINGW], [BUILDEXEEXT=".exe"] +) + case $ac_sys_system in hp*|HP*) case $CC in From 0b0a72541f5322d5b4970a36ebc708e5a8cf9a69 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jun 2023 23:42:39 +0200 Subject: [PATCH 091/123] configure.ac: fix building some test modules they are guarded by dlopen being present, but if module loading is enabled is more correct. --- configure.ac | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index b3c3ab020df37e..172c673067f534 100644 --- a/configure.ac +++ b/configure.ac @@ -5174,8 +5174,11 @@ fi AC_MSG_RESULT([$DYNLOADFILE]) if test "$DYNLOADFILE" != "dynload_stub.o" then + have_dynamic_loading=yes AC_DEFINE([HAVE_DYNAMIC_LOADING], [1], [Defined when any dynamic module loading is enabled.]) +else + have_dynamic_loading=no fi # MACHDEP_OBJS can be set to platform-specific object files needed by Python @@ -8047,20 +8050,20 @@ PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes], [], [-DPY3_DLLNAME="\"$DLLLIBRARY\""], []) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) -PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$have_dynamic_loading" = yes]) +PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$have_dynamic_loading" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testconsole], [test "$TEST_MODULES" = yes -a "$MACHDEP" = "win32"]) PY_STDLIB_MOD([_ctypes_test], - [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes], + [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$have_dynamic_loading" = yes], [], [$LIBM]) dnl Limited API template modules. dnl The limited C API is not compatible with the Py_TRACE_REFS macro. dnl Emscripten does not support shared libraries yet. -PY_STDLIB_MOD([xxlimited], [test "$with_trace_refs" = "no"], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no"], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited], [test "$with_trace_refs" = "no"], [test "$have_dynamic_loading" = yes]) +PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no"], [test "$have_dynamic_loading" = yes]) # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) From a27d2bb96d595b8464e448a9d03adc16359b8aa4 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 25 Jun 2023 17:20:52 +0530 Subject: [PATCH 092/123] Don't convert `sysconfig.get_config_var('VPATH')` to an absolute path this is required for venv to work properly --- Makefile.pre.in | 1 - configure.ac | 10 +--------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 595792fb4f29a5..1b48cd2d6dfd5f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -146,7 +146,6 @@ datarootdir= @datarootdir@ # Locations needed for semi-native fixup of sysconfig. srcdir_b2h= @srcdir_b2h@ -VPATH_b2h= @VPATH_b2h@ abs_srcdir_b2h= @abs_srcdir_b2h@ abs_builddir_b2h= @abs_builddir_b2h@ prefix_b2h= @prefix_b2h@ diff --git a/configure.ac b/configure.ac index 172c673067f534..9671d724c52e51 100644 --- a/configure.ac +++ b/configure.ac @@ -763,12 +763,8 @@ fi # without setting up the same env and also that the build of Python # itself will fail as the paths are not correct for the host tools. # -# Also, getpath.c uses GetModuleFileNameW (replacing \ with /) and -# compares that with the define VPATH (passed in via command-line) -# to determine whether it's the build- or the installed-Python. -# # To work around these issues a set of _b2h variables are created: -# VPATH_b2h, prefix_b2h, srcdir_b2h, abs_srcdir_b2h +# prefix_b2h, srcdir_b2h, abs_srcdir_b2h # and abs_builddir_b2h # .. where b2h stands for build to host. sysconfig.py replaces path # prefixes matching the non-b2h versions with the b2h equivalents. @@ -795,10 +791,6 @@ AC_DEFUN([ABS_PATH_HOST], AC_SUBST([$1]) ]) -AC_MSG_CHECKING(absolute host location of VPATH) -ABS_PATH_HOST([VPATH_b2h],[srcdir]) -AC_MSG_RESULT([$VPATH_b2h]) - AC_MSG_CHECKING(absolute host location of prefix) ABS_PATH_HOST([prefix_b2h],[prefix]) AC_MSG_RESULT([$prefix_b2h]) From e7b37fa17774cf2a2579bd4842099e29dbe03ae0 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 25 Jun 2023 17:24:11 +0530 Subject: [PATCH 093/123] Always convert `/` to `\\` before passing though pathcch functions they don't seems to handle `/` as path separator correctly --- Include/pylifecycle.h | 2 ++ Python/fileutils.c | 22 ++++++++++++++++++---- Python/pathconfig.c | 16 ++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 5d292014ff50a7..21346baf73e2ca 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -28,6 +28,8 @@ PyAPI_FUNC(char) Py_GetSepA(const char *); PyAPI_FUNC(void) Py_NormalizeSepsW(wchar_t *); PyAPI_FUNC(void) Py_NormalizeSepsA(char *); +PyAPI_FUNC(void) Py_NormalizeSepsPathcchW(wchar_t *); + /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level * exit functions. diff --git a/Python/fileutils.c b/Python/fileutils.c index 50c78f18d779dc..aae41433515bf4 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2144,19 +2144,31 @@ int _Py_isabs(const wchar_t *path) { #ifdef MS_WINDOWS + // create a copy of path and replace all forward slashes with backslashes + // pathccskiproot does not handle forward slashes + wchar_t *path_copy = _wcsdup(path); + if (path_copy == NULL) { + return 0; + } + Py_NormalizeSepsPathcchW(path_copy); + const wchar_t *tail; - HRESULT hr = PathCchSkipRoot(path, &tail); - if (FAILED(hr) || path == tail) { + HRESULT hr = PathCchSkipRoot(path_copy, &tail); + if (FAILED(hr) || path_copy == tail) { + free(path_copy); return 0; } - if (tail == &path[1] && (path[0] == SEP || path[0] == ALTSEP)) { + if (tail == &path_copy[1] && (path_copy[0] == SEP || path_copy[0] == ALTSEP)) { // Exclude paths with leading SEP + free(path_copy); return 0; } - if (tail == &path[2] && path[1] == L':') { + if (tail == &path_copy[2] && path_copy[1] == L':') { // Exclude drive-relative paths (e.g. C:filename.ext) + free(path_copy); return 0; } + free(path_copy); return 1; #else return (path[0] == SEP); @@ -2303,6 +2315,8 @@ join_relfile(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, const wchar_t *relfile) { #ifdef MS_WINDOWS + Py_NormalizeSepsPathcchW(dirname); + Py_NormalizeSepsPathcchW(relfile); if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, PATHCCH_ALLOW_LONG_PATHS))) { return -1; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 08a4a374ea6023..8053b91d8c559a 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -155,6 +155,22 @@ Py_NormalizeSepsW(wchar_t *name) } } +void +Py_NormalizeSepsPathcchW(wchar_t *name) +{ +#ifdef MS_WINDOWS + assert(name != NULL); + wchar_t sep = '\\'; + wchar_t altsep = '/'; + wchar_t* seps; + seps = wcschr(name, altsep); + while(seps) { + *seps = sep; + seps = wcschr(seps, altsep); + } +#endif +} + /* External interface */ /* Stored values set by C API functions */ From a935bf371972ffb2b85022f733f9c9b5318fb5ae Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 25 Jun 2023 17:26:55 +0530 Subject: [PATCH 094/123] Build venvlauncher.exe from PC/launcher.c - Use function available in original msvcrt.dll _wdupenv_s -> _wgetenv_s fread_s -> fread - Add a test for checking the new launchers --- Makefile.pre.in | 20 ++++++++++++-------- PC/launcher.c | 29 +++++++++++++++++------------ PC/pylauncher.rc | 20 ++++++++++---------- mingw_smoketests.py | 12 ++++++++++++ 4 files changed, 51 insertions(+), 30 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 1b48cd2d6dfd5f..2d2a64fc6ce693 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -808,6 +808,12 @@ pythonw_exe.o: $(srcdir)/PC/pythonw_exe.rc python_nt.o: $(srcdir)/PC/python_nt.rc $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ +venvlauncher.o: $(srcdir)/PC/pylauncher.rc + $(WINDRES) $(RCFLAGS) -DPY_ICON -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pylauncher.rc $@ + +venvwlauncher.o: $(srcdir)/PC/pylauncher.rc + $(WINDRES) $(RCFLAGS) -DPYW_ICON -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pylauncher.rc $@ + $(BUILDPYTHONW): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) pythonw_exe.o $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -mwindows -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) pythonw_exe.o @@ -815,13 +821,11 @@ $(BUILDPYTHONW): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) pythonw $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) python_exe.o $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) python_exe.o -# FIXME: build these from PC/launcher.c instead -$(BUILDVENVLAUNCHER): $(BUILDPYTHON) - cp $(BUILDPYTHON) $(BUILDVENVLAUNCHER) +$(BUILDVENVLAUNCHER): $(BUILDPYTHON) venvlauncher.o $(srcdir)/PC/launcher.c + $(LINKCC) -D_CONSOLE -DVENV_REDIRECT $(PY_STDMODULE_CFLAGS) -municode -static -static-libgcc -static-libstdc++ venvlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion -# FIXME: build these from PC/launcher.c instead -$(BUILDVENVWLAUNCHER): $(BUILDPYTHONW) - cp $(BUILDPYTHONW) $(BUILDVENVWLAUNCHER) +$(BUILDVENVWLAUNCHER): $(BUILDPYTHONW) venvwlauncher.o $(srcdir)/PC/launcher.c + $(LINKCC) -D_WINDOWS -DVENV_REDIRECT $(PY_STDMODULE_CFLAGS) -mwindows -municode -static -static-libgcc -static-libstdc++ venvwlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -2010,8 +2014,8 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ if test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(BINDIR)/python3w$(EXE); \ - $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(VENVLAUNCHERDIR)/python$(EXE); \ - $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(VENVLAUNCHERDIR)/pythonw$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDVENVLAUNCHER) $(DESTDIR)$(VENVLAUNCHERDIR)/python$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDVENVWLAUNCHER) $(DESTDIR)$(VENVLAUNCHERDIR)/pythonw$(EXE); \ else \ $(INSTALL_PROGRAM) $(STRIPFLAG) Mac/pythonw $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ fi diff --git a/PC/launcher.c b/PC/launcher.c index dc265533740b67..14d8b8fff83d0c 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -924,7 +924,7 @@ static COMMAND path_command; static COMMAND * find_on_path(wchar_t * name) { wchar_t * pathext; - size_t varsize; + size_t requiredSize; wchar_t * context = NULL; wchar_t * extension; COMMAND * result = NULL; @@ -941,18 +941,23 @@ static COMMAND * find_on_path(wchar_t * name) } else { /* No extension - search using registered extensions. */ - rc = _wdupenv_s(&pathext, &varsize, L"PATHEXT"); - if (rc == 0) { - extension = wcstok_s(pathext, L";", &context); - while (extension) { - len = SearchPathW(NULL, name, extension, MSGSIZE, path_command.value, NULL); - if (len) { - result = &path_command; - break; + _wgetenv_s(&requiredSize, NULL, 0, L"PATHEXT"); + if (requiredSize > 0) { + pathext = (wchar_t *)malloc(requiredSize * sizeof(wchar_t)); + /* No extension - search using registered extensions. */ + rc = _wgetenv_s(&requiredSize, pathext, requiredSize, L"PATHEXT"); + if (rc == 0) { + extension = wcstok_s(pathext, L";", &context); + while (extension) { + len = SearchPathW(NULL, name, extension, MSGSIZE, path_command.value, NULL); + if (len) { + result = &path_command; + break; + } + extension = wcstok_s(NULL, L";", &context); } - extension = wcstok_s(NULL, L";", &context); + free(pathext); } - free(pathext); } } return result; @@ -1912,7 +1917,7 @@ process(int argc, wchar_t ** argv) if (_wfopen_s(&f, venv_cfg_path, L"r")) { error(RC_BAD_VENV_CFG, L"Cannot read '%ls'", venv_cfg_path); } - cb = fread_s(buffer, sizeof(buffer), sizeof(buffer[0]), + cb = fread(buffer, sizeof(buffer[0]), sizeof(buffer) / sizeof(buffer[0]), f); fclose(f); diff --git a/PC/pylauncher.rc b/PC/pylauncher.rc index 11862643aa6989..84b291722f6633 100644 --- a/PC/pylauncher.rc +++ b/PC/pylauncher.rc @@ -12,17 +12,17 @@ 1 RT_MANIFEST "python.manifest" #if defined(PY_ICON) -1 ICON DISCARDABLE "icons\python.ico" +1 ICON DISCARDABLE "icons/python.ico" #elif defined(PYW_ICON) -1 ICON DISCARDABLE "icons\pythonw.ico" +1 ICON DISCARDABLE "icons/pythonw.ico" #else -1 ICON DISCARDABLE "icons\launcher.ico" -2 ICON DISCARDABLE "icons\py.ico" -3 ICON DISCARDABLE "icons\pyc.ico" -4 ICON DISCARDABLE "icons\pyd.ico" -5 ICON DISCARDABLE "icons\python.ico" -6 ICON DISCARDABLE "icons\pythonw.ico" -7 ICON DISCARDABLE "icons\setup.ico" +1 ICON DISCARDABLE "icons/launcher.ico" +2 ICON DISCARDABLE "icons/py.ico" +3 ICON DISCARDABLE "icons/pyc.ico" +4 ICON DISCARDABLE "icons/pyd.ico" +5 ICON DISCARDABLE "icons/python.ico" +6 ICON DISCARDABLE "icons/pythonw.ico" +7 ICON DISCARDABLE "icons/setup.ico" #endif 1 USAGE "launcher-usage.txt" @@ -64,4 +64,4 @@ BEGIN BEGIN VALUE "Translation", 0x0, 1200 END -END \ No newline at end of file +END diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 2a8bd160b0042a..00601f956d9267 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -194,6 +194,18 @@ def test_venv_creation(self): assert os.path.exists(os.path.join(tmp, "bin", "python3.exe")) subprocess.check_call([shutil.which("bash.exe"), os.path.join(tmp, "bin", "activate")]) + # This will not work in in-tree build + if not sysconfig.is_python_build(): + op = subprocess.check_output( + [ + os.path.join(tmp, "bin", "python.exe"), + "-c", + "print('Hello World')" + ], + cwd=tmp, + ) + assert op.decode().strip() == "Hello World" + def test_has_mktime(self): from time import mktime, gmtime mktime(gmtime()) From 8dd0930f7dd87b02c9616ce85098be33c6e075ce Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 25 Jun 2023 19:39:30 +0530 Subject: [PATCH 095/123] getpath.py: fix `dirname` also, fix finding prefix when in a venv --- Modules/getpath.c | 12 ++++++++++++ Modules/getpath.py | 3 +++ 2 files changed, 15 insertions(+) diff --git a/Modules/getpath.c b/Modules/getpath.c index de7efb628393e9..099cffc62568c3 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -89,6 +89,12 @@ getpath_basename(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); +#ifdef ALTSEP + if (pos < 0) { + // try using altsep + pos = PyUnicode_FindChar(path, ALTSEP, 0, end, -1); + } +#endif if (pos < 0) { return Py_NewRef(path); } @@ -105,6 +111,12 @@ getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); +#ifdef ALTSEP + if (pos < 0) { + // try using altsep + pos = PyUnicode_FindChar(path, ALTSEP, 0, end, -1); + } +#endif if (pos < 0) { return PyUnicode_FromStringAndSize(NULL, 0); } diff --git a/Modules/getpath.py b/Modules/getpath.py index 6867e98eba535f..5687e4f38de37f 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -574,6 +574,9 @@ def search_up(prefix, *landmarks, test=isfile): # First try to detect prefix by looking alongside our runtime library, if known if library and not prefix: library_dir = dirname(library) + if os_name == 'nt' and is_mingw: + # QUIRK: On Windows, mingw Python DLLs are in the bin directory + library_dir = joinpath(library_dir, '..') if ZIP_LANDMARK: if os_name == 'nt': # QUIRK: Windows does not search up for ZIP file From 1f2a2898c058cfea346990518c85f03c16a1f9f1 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 28 Jul 2023 20:13:58 +0200 Subject: [PATCH 096/123] getpath: use normpath on all generated paths Instead of just calling normpath in abspath just call normpath on all the config results. This makes sure we don't change getpath.py too much and still cover all outputs. This fixes sys.exec_prefix not matching sys.prefix, see https://github.com/msys2-contrib/cpython-mingw/issues/142 --- Lib/test/test_getpath.py | 6 ++++++ Modules/getpath.c | 21 ++++++++++++++++++++- Modules/getpath.py | 30 ++++++++++++++++-------------- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 4dc2d305730d67..f63e4204a78a2e 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -913,6 +913,9 @@ def __missing__(self, key): except AttributeError: raise KeyError(key) from None + def normpath(self, path): + return ntpath.normpath(path) + def abspath(self, path): if self.isabs(path): return path @@ -1091,6 +1094,9 @@ def __missing__(self, key): except AttributeError: raise KeyError(key) from None + def normpath(self, path): + return path + def abspath(self, path): if self.isabs(path): return path diff --git a/Modules/getpath.c b/Modules/getpath.c index 099cffc62568c3..47b89ebcf522ed 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -54,6 +54,25 @@ /* HELPER FUNCTIONS for getpath.py */ +static PyObject * +getpath_normpath(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } + Py_ssize_t len; + wchar_t *buffer = PyUnicode_AsWideCharString(pathobj, &len); + if (!buffer) { + return NULL; + } + r = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + PyMem_Free(buffer); + return r; +} + static PyObject * getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) { @@ -68,7 +87,6 @@ getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) if (path) { wchar_t *abs; if (_Py_abspath((const wchar_t *)_Py_normpath(path, -1), &abs) == 0 && abs) { - abs = _Py_normpath(abs, -1); r = PyUnicode_FromWideChar(abs, -1); PyMem_RawFree((void *)abs); } else { @@ -523,6 +541,7 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) static PyMethodDef getpath_methods[] = { + {"normpath", getpath_normpath, METH_VARARGS, NULL}, {"abspath", getpath_abspath, METH_VARARGS, NULL}, {"basename", getpath_basename, METH_VARARGS, NULL}, {"dirname", getpath_dirname, METH_VARARGS, NULL}, diff --git a/Modules/getpath.py b/Modules/getpath.py index 5687e4f38de37f..d143f9776eff31 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -233,6 +233,8 @@ def search_up(prefix, *landmarks, test=isfile): return prefix prefix = dirname(prefix) +def _normpath(p): + return normpath(p) if p is not None else None # ****************************************************************************** # READ VARIABLES FROM config @@ -670,7 +672,7 @@ def search_up(prefix, *landmarks, test=isfile): if py_setpath: # If Py_SetPath was called then it overrides any existing search path - config['module_search_paths'] = py_setpath.split(DELIM) + config['module_search_paths'] = [_normpath(p) for p in py_setpath.split(DELIM)] config['module_search_paths_set'] = 1 elif not pythonpath_was_set: @@ -757,7 +759,7 @@ def search_up(prefix, *landmarks, test=isfile): if platstdlib_dir: pythonpath.append(platstdlib_dir) - config['module_search_paths'] = pythonpath + config['module_search_paths'] = [_normpath(p) for p in pythonpath] config['module_search_paths_set'] = 1 @@ -792,23 +794,23 @@ def search_up(prefix, *landmarks, test=isfile): warn("unsupported 'import' line in ._pth file") else: pythonpath.append(joinpath(pth_dir, line)) - config['module_search_paths'] = pythonpath + config['module_search_paths'] = [_normpath(p) for p in pythonpath] config['module_search_paths_set'] = 1 # ****************************************************************************** # UPDATE config FROM CALCULATED VALUES # ****************************************************************************** -config['program_name'] = program_name -config['home'] = home -config['executable'] = executable -config['base_executable'] = base_executable -config['prefix'] = prefix -config['exec_prefix'] = exec_prefix -config['base_prefix'] = base_prefix or prefix -config['base_exec_prefix'] = base_exec_prefix or exec_prefix +config['program_name'] = _normpath(program_name) +config['home'] = _normpath(home) +config['executable'] = _normpath(executable) +config['base_executable'] = _normpath(base_executable) +config['prefix'] = _normpath(prefix) +config['exec_prefix'] = _normpath(exec_prefix) +config['base_prefix'] = _normpath(base_prefix or prefix) +config['base_exec_prefix'] = _normpath(base_exec_prefix or exec_prefix) -config['platlibdir'] = platlibdir +config['platlibdir'] = _normpath(platlibdir) # test_embed expects empty strings, not None -config['stdlib_dir'] = stdlib_dir or '' -config['platstdlib_dir'] = platstdlib_dir or '' +config['stdlib_dir'] = _normpath(stdlib_dir or '') +config['platstdlib_dir'] = _normpath(platstdlib_dir or '') From b0908f8b64b4527994119d65f949ae1305a25d8e Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 28 Jul 2023 20:14:38 +0200 Subject: [PATCH 097/123] pathconfig: normpath sys.path[0] this gets added as first element in sys.path. use normpath to make sure it uses the right path separator --- Python/pathconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 8053b91d8c559a..7eb9006be5b40a 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -662,7 +662,7 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p) } #endif /* All others */ - PyObject *path0_obj = PyUnicode_FromWideChar(path0, n); + PyObject *path0_obj = PyUnicode_FromWideChar(_Py_normpath(path0, -1), n); if (path0_obj == NULL) { return -1; } From 38612d894aaf3266a14e29e9508bc14c663b8f96 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 28 Jul 2023 20:15:01 +0200 Subject: [PATCH 098/123] smoketests: add some tests for sys/site paths --- mingw_smoketests.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 00601f956d9267..516005c9b3eeb6 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -238,6 +238,29 @@ def test_platform_things(self): self.assertEqual(platform.system(), "Windows") self.assertTrue(isinstance(sys.api_version, int) and sys.api_version > 0) + def test_sys_getpath(self): + # everything sourced from getpath.py + import sys + + def assertNormpath(path): + self.assertEqual(path, os.path.normpath(path)) + + assertNormpath(sys.executable) + assertNormpath(sys._base_executable) + assertNormpath(sys.prefix) + assertNormpath(sys.base_prefix) + assertNormpath(sys.exec_prefix) + assertNormpath(sys.base_exec_prefix) + assertNormpath(sys.platlibdir) + assertNormpath(sys._stdlib_dir) + for p in sys.path: + assertNormpath(p) + + def test_site(self): + import site + + self.assertEqual(len(site.getsitepackages()), 1) + def test_c_ext_build(self): import tempfile import sys From 066c14bd214d7da59aef8385aab8c8a4c6ef13e9 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sat, 5 Aug 2023 14:10:27 +0530 Subject: [PATCH 099/123] Search DLLs only on paths added using `add_dll_directory()`. This is the default behavior on upstream Python. We previously reverted this behavior because it broke some use cases. The old behavior can be restored by setting the environment variable PYTHONLEGACYWINDOWSDLLLOADING to 1. Fixes https://github.com/msys2-contrib/cpython-mingw/issues/48 Also fixes https://github.com/msys2-contrib/cpython-mingw/issues/141 Signed-off-by: Naveen M K --- .github/workflows/mingw.yml | 10 +++++----- Lib/test/__main__.py | 8 ++++++++ Python/dynload_win.c | 16 ++++++++++++++-- Tools/build/check_extension_modules.py | 4 ++++ mingw_smoketests.py | 3 +++ 5 files changed, 34 insertions(+), 7 deletions(-) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 2b606fb992dd0e..0c9f644d84cca5 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -124,15 +124,15 @@ jobs: cp "${pkgdir}${MINGW_PREFIX}"/bin/idle3 "${pkgdir}${MINGW_PREFIX}"/bin/idle cp "${pkgdir}${MINGW_PREFIX}"/bin/pydoc3 "${pkgdir}${MINGW_PREFIX}"/bin/pydoc + # copy to / + cp -rf "${pkgdir}"/* / + - name: Run Smoke Test (installed) shell: msys2 {0} run: | - export PYTHONTZPATH="${MINGW_PREFIX}/share/zoneinfo" SMOKETESTS="$(pwd)/mingw_smoketests.py" - cd _build - cd python_pkgdir/${MINGW_PREFIX}/bin - ./python.exe "$SMOKETESTS" - MSYSTEM= ./python.exe "$SMOKETESTS" + ${MINGW_PREFIX}/bin/python.exe "$SMOKETESTS" + MSYSTEM= ${MINGW_PREFIX}/bin/python.exe "$SMOKETESTS" - name: Compress if: always() diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index 82b50ad2c6e777..25bac39e24af20 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,2 +1,10 @@ +import os +import sys + from test.libregrtest.main import main + +if sys.platform == "win32": + # Enable DLL loading from PATH. + os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" + main(_add_python_opts=True) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 1f90e21a77fecc..db7968742b2228 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_add_relfile() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_initconfig.h" #ifdef HAVE_DIRECT_H #include @@ -225,6 +226,18 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, dl_funcptr p; char funcname[258], *import_python; + int use_legacy = 0; + DWORD load_library_flags = 0; + + _Py_get_env_flag(1, &use_legacy, "PYTHONLEGACYWINDOWSDLLLOADING"); + + if (use_legacy) { + load_library_flags = LOAD_WITH_ALTERED_SEARCH_PATH; + } else { + load_library_flags = LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR; + } + #ifdef _MSC_VER _Py_CheckPython3(); #endif /* _MSC_VER */ @@ -249,8 +262,7 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, AddDllDirectory function. We add SEARCH_DLL_LOAD_DIR to ensure DLLs adjacent to the PYD are preferred. */ Py_BEGIN_ALLOW_THREADS - hDLL = LoadLibraryExW(wpathname, NULL, - LOAD_WITH_ALTERED_SEARCH_PATH); + hDLL = LoadLibraryExW(wpathname, NULL, load_library_flags); Py_END_ALLOW_THREADS PyMem_Free(wpathname); diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index aa1ade71a8e937..7ded0f464c205d 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -61,6 +61,10 @@ "winsound", } +if sys.platform == "win32": + # Enable DLL loading from PATH. + os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" + logger = logging.getLogger(__name__) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 516005c9b3eeb6..ca1f652caeec27 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -34,6 +34,9 @@ else: SEP = "\\" +if sysconfig.is_python_build(): + os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" + _UCRT = sysconfig.get_platform() not in ('mingw_x86_64', 'mingw_i686') From 7347577f01a58a37309bced4ff68095f4945c991 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Mon, 21 Aug 2023 08:19:28 +0200 Subject: [PATCH 100/123] Build and install libpython3.dll This provides the limited ABI as a separate DLL, forwarding to the real one. This makes linking with "-lpython3" work. Fixes #147 --- Makefile.pre.in | 15 +++++++++++++-- PC/python3dll.c | 8 ++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2d2a64fc6ce693..e9ded19a3fb519 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -273,6 +273,8 @@ LIBRARY_DEPS= @LIBRARY_DEPS@ LINK_PYTHON_DEPS=@LINK_PYTHON_DEPS@ PY_ENABLE_SHARED= @PY_ENABLE_SHARED@ STATIC_LIBPYTHON= @STATIC_LIBPYTHON@ +ABI3DLLLIBRARY= libpython3.dll +ABI3LDLIBRARY= libpython3.dll.a LIBS= @LIBS@ @@ -633,7 +635,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: build_all build_all: check-clean-src $(BUILDPYTHON) $(BUILDPYTHONW) $(BUILDVENVLAUNCHER) $(BUILDVENVWLAUNCHER) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil $(ABI3DLLLIBRARY) $(ABI3LDLIBRARY) .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ @@ -808,6 +810,9 @@ pythonw_exe.o: $(srcdir)/PC/pythonw_exe.rc python_nt.o: $(srcdir)/PC/python_nt.rc $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ +python3dll_nt.o: $(srcdir)/PC/python_nt.rc + $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(ABI3DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ + venvlauncher.o: $(srcdir)/PC/pylauncher.rc $(WINDRES) $(RCFLAGS) -DPY_ICON -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pylauncher.rc $@ @@ -921,6 +926,10 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) python_nt.o else true; \ fi +$(ABI3DLLLIBRARY) $(ABI3LDLIBRARY): python3dll_nt.o $(srcdir)/PC/launcher.c + $(LDSHARED) -DPYTHON_DLL_NAME=\"$(DLLLIBRARY)\" $(srcdir)/PC/python3dll.c -Wl,--out-implib=$(ABI3LDLIBRARY) -o $(ABI3DLLLIBRARY) python3dll_nt.o \ + $(LDFLAGS_NODIST); + # wasm32-emscripten browser build # wasm assets directory is relative to current build dir, e.g. "./usr/local". # --preload-file turns a relative asset path into an absolute path. @@ -2029,6 +2038,7 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ if test -f $(LDLIBRARY) && test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ if test -n "$(DLLLIBRARY)" ; then \ $(INSTALL_SHARED) $(DLLLIBRARY) $(DESTDIR)$(BINDIR); \ + $(INSTALL_SHARED) $(ABI3DLLLIBRARY) $(DESTDIR)$(BINDIR); \ else \ $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(LIBDIR)/$(INSTSONAME); \ if test $(LDLIBRARY) != $(INSTSONAME); then \ @@ -2506,6 +2516,7 @@ libainstall: all scripts if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ if test "$(SHLIB_SUFFIX)" = .dll -o "$(SHLIB_SUFFIX)" = .pyd; then \ $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBDIR) ; \ + $(INSTALL_DATA) $(ABI3LDLIBRARY) $(DESTDIR)$(LIBDIR) ; \ else \ $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \ fi; \ @@ -2759,7 +2770,7 @@ clean: clean-retain-profile clean-bolt .PHONY: clobber clobber: clean - -rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ + -rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY)$(ABI3LDLIBRARY) $(ABI3DLLLIBRARY) \ tags TAGS \ config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform diff --git a/PC/python3dll.c b/PC/python3dll.c index 505feef4b986c4..ee9380c81437e8 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -2,6 +2,7 @@ /* Generated by Tools/build/stable_abi.py */ +#ifdef _MSC_VER #ifdef _M_IX86 #define DECORATE "_" #else @@ -12,6 +13,13 @@ __pragma(comment(linker, "/EXPORT:" DECORATE #name "=" PYTHON_DLL_NAME "." #name)) #define EXPORT_DATA(name) \ __pragma(comment(linker, "/EXPORT:" DECORATE #name "=" PYTHON_DLL_NAME "." #name ",DATA")) +#else +// XXX: Why do we need the .dll extension and no DECORATE compared to the MSVC case? +#define EXPORT_FUNC(name) \ + asm(".section .drectve\n\t.ascii \" -export:" #name "=\\\"" PYTHON_DLL_NAME "." #name "\\\" \""); +#define EXPORT_DATA(name) \ + asm(".section .drectve\n\t.ascii \" -export:" #name "=\\\"" PYTHON_DLL_NAME "." #name "\\\",DATA \""); +#endif EXPORT_FUNC(_Py_BuildValue_SizeT) EXPORT_FUNC(_Py_CheckRecursiveCall) From f78fc1f81a1082d05845367702a26cb853ead1cc Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 27 Aug 2023 15:00:32 +0200 Subject: [PATCH 101/123] Port GetPythonImport() to mingw This looks for DLL names in the import table but while with MSVC the DLL is named python311.dll in our case it is named libpython3.11.dll. Adjust the strings and lengths accordingly. --- Python/dynload_win.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index db7968742b2228..36d1c7a0353968 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -56,6 +56,16 @@ const char *_PyImport_DynLoadFiletab[] = { #define DWORD_AT(mem) (*(DWORD *)(mem)) #define WORD_AT(mem) (*(WORD *)(mem)) +#ifdef __MINGW32__ +#define DLL_PREFIX "libpython" +#define DLL_PREFIX_LEN 9 +#define DLL_NAME_FORMAT "libpython%d.%d" +#else +#define DLL_PREFIX "python" +#define DLL_PREFIX_LEN 6 +#define DLL_NAME_FORMAT "python%d%d" +#endif + static char *GetPythonImport (HINSTANCE hModule) { unsigned char *dllbase, *import_data, *import_name; @@ -122,15 +132,15 @@ static char *GetPythonImport (HINSTANCE hModule) import_off); while (DWORD_AT(import_data)) { import_name = dllbase + DWORD_AT(import_data+12); - if (strlen(import_name) >= 6 && - !strncmp(import_name,"python",6)) { + if (strlen(import_name) >= DLL_PREFIX_LEN && + !strncmp(import_name, DLL_PREFIX, DLL_PREFIX_LEN)) { char *pch; /* Don't claim that python3.dll is a Python DLL. */ #ifdef _DEBUG - if (strcmp(import_name, "python3_d.dll") == 0) { + if (strcmp(import_name, DLL_PREFIX "3_d.dll") == 0) { #else - if (strcmp(import_name, "python3.dll") == 0) { + if (strcmp(import_name, DLL_PREFIX "3.dll") == 0) { #endif import_data += 20; continue; @@ -138,7 +148,7 @@ static char *GetPythonImport (HINSTANCE hModule) /* Ensure python prefix is followed only by numbers to the end of the basename */ - pch = import_name + 6; + pch = import_name + DLL_PREFIX_LEN; #ifdef _DEBUG while (*pch && pch[0] != '_' && pch[1] != 'd' && pch[2] != '.') { #else @@ -329,9 +339,9 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, PyOS_snprintf(buffer, sizeof(buffer), #ifdef _DEBUG - "python%d%d_d.dll", + DLL_NAME_FORMAT "_d.dll", #else - "python%d%d.dll", + DLL_NAME_FORMAT ".dll", #endif PY_MAJOR_VERSION,PY_MINOR_VERSION); import_python = GetPythonImport(hDLL); From 8d63ba224d93b0c130a08c8b149ca53fafbb16a2 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 27 Aug 2023 16:04:50 +0200 Subject: [PATCH 102/123] LoadLibraryExW: make sure to only use backslashes for paths It seems like in case the path passed to it is absolute, but contains forward slashes then LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR does not work and DLLs in the same directory as the extension are not considered. This occurs in our fork because in MSYS2-mode the extension loader will normalize to forward slashes before. Normalize everything to backslashes again before passing it to LoadLibraryExW. Fixes #151 --- Python/dynload_win.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 36d1c7a0353968..caa49681d175e9 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -252,10 +252,22 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, _Py_CheckPython3(); #endif /* _MSC_VER */ +// So we can adjust the separators in the path below +#define USE_UNICODE_WCHAR_CACHE 0 + wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL); if (wpathname == NULL) return NULL; + // LoadLibraryExW only considers paths using backslashes as "fully qualified", + // and for example LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR doesn't work with forward slashes. + // https://github.com/msys2-contrib/cpython-mingw/issues/151 + for (size_t i = 0; wpathname[i] != L'\0'; ++i) { + if (wpathname[i] == L'/') { + wpathname[i] = L'\\'; + } + } + PyOS_snprintf(funcname, sizeof(funcname), "%.20s_%.200s", prefix, shortname); { From 34d859b152e9b56808b6fb4dac252691ea0c42a8 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 8 Nov 2023 19:44:22 +0530 Subject: [PATCH 103/123] mingw_smoketests: add a test to check if sysconfig returns correct values for `LIBDIR` --- mingw_smoketests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index ca1f652caeec27..4f81c68803199c 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -218,6 +218,9 @@ def test_platform_things(self): import sysconfig import platform import importlib.machinery + import tempfile + import venv + import subprocess self.assertEqual(sys.implementation.name, "cpython") self.assertEqual(sys.platform, "win32") self.assertTrue(sysconfig.get_platform().startswith("mingw")) @@ -241,6 +244,21 @@ def test_platform_things(self): self.assertEqual(platform.system(), "Windows") self.assertTrue(isinstance(sys.api_version, int) and sys.api_version > 0) + with tempfile.TemporaryDirectory() as tmp: + builder = venv.EnvBuilder() + builder.create(tmp) + # This will not work in in-tree build + if not sysconfig.is_python_build(): + op = subprocess.check_output( + [ + os.path.join(tmp, "bin", "python.exe"), + "-c", + "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))" + ], + cwd=tmp, + ) + self.assertTrue(op.decode().strip().startswith(sys.base_prefix)) + def test_sys_getpath(self): # everything sourced from getpath.py import sys From 618b83dd7e78b116c2f8e6f6d1b9c3491f436150 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 13:42:01 +0530 Subject: [PATCH 104/123] Add missing library for multiprocessing module `ws2_32` is required when building that module --- configure.ac | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 9671d724c52e51..0a82add8eb061a 100644 --- a/configure.ac +++ b/configure.ac @@ -4282,6 +4282,13 @@ AS_CASE([$MACHDEP], [SOCKET_LIBS=""] ) +dnl On MINGW, you need to link againt ws2_32 for _multiprocessing +AS_CASE([$MACHDEP], + [win32], [MULTIPROCESSING_LIBS="-lws2_32"], + [MULTIPROCESSING_LIBS=""] +) + + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -7899,7 +7906,7 @@ PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules PY_STDLIB_MOD([_multiprocessing], [], [test "$ac_cv_func_sem_unlink" = "yes" -o "$MACHDEP" = "win32"], - [-I\$(srcdir)/Modules/_multiprocessing]) + [-I\$(srcdir)/Modules/_multiprocessing], [$MULTIPROCESSING_LIBS]) PY_STDLIB_MOD([_posixshmem], [], [test "$have_posix_shmem" = "yes"], [$POSIXSHMEM_CFLAGS], [$POSIXSHMEM_LIBS]) From 0f110546b3a74c53235fe0aa68b6d4ee8503932c Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 14:14:45 +0530 Subject: [PATCH 105/123] selectmodule: add support for MINGW --- Modules/selectmodule.c | 6 +++--- configure.ac | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 50788e5344c152..89396f31d0daf3 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -169,9 +169,9 @@ seq2set(PyObject *seq, fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) v = PyObject_AsFileDescriptor( o ); if (v == -1) goto finally; -#if defined(_MSC_VER) +#if defined(MS_WIN32) max = 0; /* not used for Win32 */ -#else /* !_MSC_VER */ +#else /* !MS_WIN32 */ if (!_PyIsSelectable_fd(v)) { PyErr_SetString(PyExc_ValueError, "filedescriptor out of range in select()"); @@ -179,7 +179,7 @@ seq2set(PyObject *seq, fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) } if (v > max) max = v; -#endif /* _MSC_VER */ +#endif /* MS_WIN32 */ FD_SET(v, set); /* add object and its file descriptor to the list */ diff --git a/configure.ac b/configure.ac index 0a82add8eb061a..353ff024d67e9f 100644 --- a/configure.ac +++ b/configure.ac @@ -4288,6 +4288,11 @@ AS_CASE([$MACHDEP], [MULTIPROCESSING_LIBS=""] ) +dnl On MINGW, you need to link againt ws2_32 for selectmodule +AS_CASE([$MACHDEP], + [win32], [SELECTMODULE_LIBS="-lws2_32"], + [SELECTMODULE_LIBS=""] +) dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -7896,7 +7901,7 @@ PY_STDLIB_MOD_SIMPLE([_opcode]) PY_STDLIB_MOD_SIMPLE([_pickle]) PY_STDLIB_MOD_SIMPLE([_queue]) PY_STDLIB_MOD_SIMPLE([_random]) -PY_STDLIB_MOD_SIMPLE([select]) +PY_STDLIB_MOD_SIMPLE([select], [], [$SELECTMODULE_LIBS]) PY_STDLIB_MOD_SIMPLE([_struct]) PY_STDLIB_MOD_SIMPLE([_typing]) PY_STDLIB_MOD_SIMPLE([_xxsubinterpreters]) From 1e2a68ccfaa17a9b445a780e59c3453cac048741 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 14:19:41 +0530 Subject: [PATCH 106/123] readline: disable on MINGW --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 353ff024d67e9f..e15e19af4ef66c 100644 --- a/configure.ac +++ b/configure.ac @@ -7760,6 +7760,7 @@ AS_CASE([$ac_sys_system], [VxWorks*], [PY_STDLIB_MOD_SET_NA([_scproxy], [_crypt], [termios], [grp])], [Darwin], [PY_STDLIB_MOD_SET_NA([ossaudiodev], [spwd])], [CYGWIN*], [PY_STDLIB_MOD_SET_NA([_scproxy], [nis])], + [MINGW*], [PY_STDLIB_MOD_SET_NA([readline])], [QNX*], [PY_STDLIB_MOD_SET_NA([_scproxy], [nis])], [FreeBSD*], [PY_STDLIB_MOD_SET_NA([_scproxy], [spwd])], [Emscripten|WASI], [ From 6fa96e4de4febe8a4c5d91f33356f704cf06e425 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 14:25:11 +0530 Subject: [PATCH 107/123] Add libraries to fix compile of ctypes on MINGW --- configure.ac | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index e15e19af4ef66c..159d4feafab43c 100644 --- a/configure.ac +++ b/configure.ac @@ -4294,6 +4294,12 @@ AS_CASE([$MACHDEP], [SELECTMODULE_LIBS=""] ) +dnl On MINGW, you need to link againt ole32, oleaut32 and uuid for ctypes +AS_CASE([$MACHDEP], + [win32], [CTYPES_LIBS="-lole32 -loleaut32 -luuid"], + [CTYPES_LIBS=""] +) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -7991,7 +7997,7 @@ PY_STDLIB_MOD([_crypt], [$LIBCRYPT_CFLAGS], [$LIBCRYPT_LIBS]) PY_STDLIB_MOD([_ctypes], [], [test "$have_libffi" = yes], - [$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS], [$LIBFFI_LIBS]) + [$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS], [$LIBFFI_LIBS $CTYPES_LIBS]) PY_STDLIB_MOD([_curses], [], [test "$have_curses" != "no"], [$CURSES_CFLAGS], [$CURSES_LIBS] @@ -8062,7 +8068,7 @@ PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testconsole], [test "$TEST_MODULES" = yes -a "$MACHDEP" = "win32"]) PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$have_dynamic_loading" = yes], - [], [$LIBM]) + [], [$LIBM $CTYPES_LIBS]) dnl Limited API template modules. dnl The limited C API is not compatible with the Py_TRACE_REFS macro. From 6fbd5316a0484f3f424f6e9d4b7a5ea6a63f785d Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 14:48:28 +0530 Subject: [PATCH 108/123] Add `-lpython` while linking stdlib Generalize the `MINGW` case. In theory, it should be correct for other system too and would fix cross-compilation --- Modules/makesetup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/makesetup b/Modules/makesetup index f000c9cd67310e..5e2ada098a5452 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -88,7 +88,7 @@ NL='\ ' # Setup to link with extra libraries when making shared extensions. -# Currently, only Cygwin needs this baggage. +# Currently, only Cygwin and MINGW needs this baggage. case `uname -s` in CYGWIN*) if test $libdir = . then @@ -97,6 +97,9 @@ CYGWIN*) if test $libdir = . ExtraLibDir='$(LIBPL)' fi ExtraLibs="-L$ExtraLibDir -lpython\$(LDVERSION)";; +*) + ExtraLibs='$(BLDLIBRARY)' + ExtraLibDepends='$(LIBRARY_DEPS)';; esac # Main loop @@ -285,7 +288,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | BUILT_SHARED="$BUILT_SHARED $mod" ;; esac - rule="$file: $objs" + rule="$file: $objs $ExtraLibDepends" rule="$rule; \$(BLDSHARED) $objs $libs $ExtraLibs -o $file" echo "$rule" >>$rulesf done From 6843f1f35a166cc5c706c45802418abfcb8096c6 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 15:04:43 +0530 Subject: [PATCH 109/123] Enable `_uuid` on MINGW --- configure.ac | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 159d4feafab43c..2575149cfc8956 100644 --- a/configure.ac +++ b/configure.ac @@ -4300,6 +4300,12 @@ AS_CASE([$MACHDEP], [CTYPES_LIBS=""] ) +dnl On MINGW, you need to link againt rpcrt4 for _uuid +AS_CASE([$MACHDEP], + [win32], [UUID_LIBS="-lrpcrt4"], + [UUID_LIBS=""] +) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -8027,8 +8033,8 @@ PY_STDLIB_MOD([_tkinter], [], [test "$have_tcltk" = "yes"], [$TCLTK_CFLAGS], [$TCLTK_LIBS]) PY_STDLIB_MOD([_uuid], - [], [test "$have_uuid" = "yes"], - [$LIBUUID_CFLAGS], [$LIBUUID_LIBS]) + [], [test "$have_uuid" = "yes" -o "$MACHDEP" = "win32"], + [$LIBUUID_CFLAGS], [$LIBUUID_LIBS $UUID_LIBS]) dnl compression libs PY_STDLIB_MOD([zlib], [], [test "$have_zlib" = yes], From ffe1a51f5955750236e1ec2da72f0e105a7c74af Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 16:43:41 +0530 Subject: [PATCH 110/123] Enable curses library on MINGW --- configure.ac | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configure.ac b/configure.ac index 2575149cfc8956..c4dfe3b46825da 100644 --- a/configure.ac +++ b/configure.ac @@ -6868,6 +6868,14 @@ dnl have_panel=[no, panelw, panel] have_curses=no have_panel=no +case "$host_os" in + mingw*) + # On Mingw, include directory is different, so just assume + # it's there. + ac_cv_header_ncurses_h=yes + ;; +esac + AH_TEMPLATE([HAVE_NCURSESW], [Define to 1 if you have the `ncursesw' library.]) AC_CHECK_HEADERS([curses.h ncurses.h]) From 81fdef62bb35c3a66a8615f811c5d055e7d3acc6 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 2 Feb 2024 14:43:29 +0530 Subject: [PATCH 111/123] mingw_smoketests: disable distutils related tests --- mingw_smoketests.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 4f81c68803199c..0ab5d3b60a443e 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -61,24 +61,24 @@ def test_userdir_path_sep(self): self.assertEqual(str(Path("~").expanduser()), expanduser("~")) self.assertEqual(str(Path.home()), expanduser("~")) - def test_sysconfig_schemes(self): - # https://github.com/msys2/MINGW-packages/issues/9319 - import sysconfig - from distutils.dist import Distribution - from distutils.command.install import install - - names = ['scripts', 'purelib', 'platlib', 'data', 'include'] - for scheme in ["nt", "nt_user"]: - for name in names: - c = install(Distribution({"name": "foobar"})) - c.user = (scheme == "nt_user") - c.finalize_options() - if name == "include": - dist_path = os.path.dirname(getattr(c, "install_" + "headers")) - else: - dist_path = getattr(c, "install_" + name) - sys_path = sysconfig.get_path(name, scheme) - self.assertEqual(dist_path, sys_path, (scheme, name)) + # def test_sysconfig_schemes(self): + # # https://github.com/msys2/MINGW-packages/issues/9319 + # import sysconfig + # from distutils.dist import Distribution + # from distutils.command.install import install + + # names = ['scripts', 'purelib', 'platlib', 'data', 'include'] + # for scheme in ["nt", "nt_user"]: + # for name in names: + # c = install(Distribution({"name": "foobar"})) + # c.user = (scheme == "nt_user") + # c.finalize_options() + # if name == "include": + # dist_path = os.path.dirname(getattr(c, "install_" + "headers")) + # else: + # dist_path = getattr(c, "install_" + name) + # sys_path = sysconfig.get_path(name, scheme) + # self.assertEqual(dist_path, sys_path, (scheme, name)) def test_ctypes_find_library(self): from ctypes.util import find_library From ce67dc2ec032ab5c4092bd37bb7f801052ba2b4a Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 19 Feb 2024 22:09:25 +0530 Subject: [PATCH 112/123] configure.ac: use autoconf 2.69 llvm-mingw only has autoconf 2.69 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c4dfe3b46825da..fe4d39fc879a97 100644 --- a/configure.ac +++ b/configure.ac @@ -12,7 +12,7 @@ dnl # Set VERSION so we only need to edit in one place (i.e., here) m4_define([PYTHON_VERSION], [3.12]) -AC_PREREQ([2.71]) +AC_PREREQ([2.69]) AC_INIT([python],[PYTHON_VERSION],[https://github.com/python/cpython/issues/]) From 494b9665344f3ffe595db5a2225b7ee6fb0fa45a Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Feb 2024 11:04:39 +0530 Subject: [PATCH 113/123] CI: clean up ignored tests Remove tests which are passing now, or no longer exist, and move tests which only fail with msvcrt into a separate file. --- .github/workflows/mingw.yml | 12 ++++++++++-- Lib/test/test_wmi.py | 2 +- mingw_ignorefile.txt | 24 +++--------------------- mingw_ignorefile_msvcrt.txt | 8 ++++++++ 4 files changed, 22 insertions(+), 24 deletions(-) create mode 100644 mingw_ignorefile_msvcrt.txt diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 0c9f644d84cca5..827dfe5a3705e5 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -89,16 +89,24 @@ jobs: shell: msys2 {0} run: | IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + IGNOREFILE_EXTRA="$IGNOREFILE" + if [[ "${{ matrix.msystem }}" == "MINGW32" ]] || [[ "${{ matrix.msystem }}" == "MINGW64" ]]; then + IGNOREFILE_EXTRA="$(pwd)/mingw_ignorefile_msvcrt.txt" + fi cd _build - MSYSTEM= ./python.exe -m test -j8 --ignorefile "$IGNOREFILE" -W + MSYSTEM= ./python.exe -m test -j8 --ignorefile "$IGNOREFILE" --ignorefile "$IGNOREFILE_EXTRA" -W - name: Run broken tests continue-on-error: true shell: msys2 {0} run: | IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + IGNOREFILE_EXTRA="$IGNOREFILE" + if [[ "${{ matrix.msystem }}" == "MINGW32" ]] || [[ "${{ matrix.msystem }}" == "MINGW64" ]]; then + IGNOREFILE_EXTRA="$(pwd)/mingw_ignorefile_msvcrt.txt" + fi cd _build - MSYSTEM= ./python.exe -m test -j8 --matchfile "$IGNOREFILE" -W + MSYSTEM= ./python.exe -m test -j8 --matchfile "$IGNOREFILE" --matchfile "$IGNOREFILE_EXTRA" -W - name: Install shell: msys2 {0} diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index 111f990656ced5..3dc452e79fe388 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -8,7 +8,7 @@ # Do this first so test will be skipped if module doesn't exist -_wmi = import_helper.import_module('_wmi', required_on=['win']) +_wmi = import_helper.import_module('_wmi') def wmi_exec_query(query): diff --git a/mingw_ignorefile.txt b/mingw_ignorefile.txt index e692d7ff26eb4d..550e72413a4f4e 100644 --- a/mingw_ignorefile.txt +++ b/mingw_ignorefile.txt @@ -2,16 +2,11 @@ test.test_ctypes.test_loading.LoaderTest.test_load_dll_with_flags distutils.tests.test_bdist_dumb.BuildDumbTestCase.test_simple_built distutils.tests.test_cygwinccompiler.CygwinCCompilerTestCase.test_get_versions distutils.tests.test_util.UtilTestCase.test_change_root -test.datetimetester.TestLocalTimeDisambiguation_Fast.* -test.datetimetester.TestLocalTimeDisambiguation_Pure.* -test.test_cmath.CMathTests.test_specific_values -test.test_cmd_line_script.CmdLineTest.test_consistent_sys_path_for_direct_execution test.test_compileall.CommandLineTestsNoSourceEpoch.* test.test_compileall.CommandLineTestsWithSourceEpoch.* test.test_compileall.CompileallTestsWithoutSourceEpoch.* test.test_compileall.CompileallTestsWithSourceEpoch.* test.test_import.ImportTests.test_dll_dependency_import -test.test_math.MathTests.* test.test_ntpath.NtCommonTest.test_import test.test_os.StatAttributeTests.test_stat_block_device test.test_os.TestScandir.test_attributes @@ -23,21 +18,8 @@ test.test_site._pthFileTests.* test.test_site.HelperFunctionsTests.* test.test_site.StartupImportTests.* test.test_ssl.* -test.test_strptime.CalculationTests.* -test.test_strptime.StrptimeTests.test_weekday -test.test_strptime.TimeRETests.test_compile -test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date test.test_venv.BasicTest.* test.test_venv.EnsurePipTest.* -test.test_sysconfig.TestSysConfig.test_user_similar -test.test_tcl.TclTest.testLoadWithUNC -test.test_wmi -# flaky -test.test__xxsubinterpreters.* -test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe -test.test_asynchat.TestAsynchat.test_line_terminator2 -test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_gc_aclose_09 -test.test_concurrent_futures.ThreadPoolShutdownTest.test_interpreter_shutdown -test.test_asynchat.TestNotConnected.test_disallow_negative_terminator -test.test_logging.SysLogHandlerTest.* -test.test_logging.IPv6SysLogHandlerTest.* + +test.test_dict.DictTest.test_splittable_to_generic_combinedtable +test.test_capi.test_run.CAPITest.test_run_fileexflags diff --git a/mingw_ignorefile_msvcrt.txt b/mingw_ignorefile_msvcrt.txt new file mode 100644 index 00000000000000..be363bd0faf404 --- /dev/null +++ b/mingw_ignorefile_msvcrt.txt @@ -0,0 +1,8 @@ +test.datetimetester.TestLocalTimeDisambiguation_Fast.* +test.datetimetester.TestLocalTimeDisambiguation_Pure.* +test.test_cmath.CMathTests.test_specific_values +test.test_math.MathTests.* +test.test_strptime.CalculationTests.* +test.test_strptime.StrptimeTests.test_weekday +test.test_strptime.TimeRETests.test_compile +test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date From 8e9d8cda56d3cfb50ff6d68f135eae657856e8bf Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Thu, 11 Jul 2024 13:58:20 +0530 Subject: [PATCH 114/123] gcc: make `incompatible-pointer-types` as warning gcc 14 changed `incompatible-pointer-types` from warning to error for now, make it as warning to avoid build failure --- configure.ac | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configure.ac b/configure.ac index fe4d39fc879a97..5d1b6e25269823 100644 --- a/configure.ac +++ b/configure.ac @@ -2528,6 +2528,14 @@ case $GCC in yes) CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" + # GCC 14 and later error out with incompatible-pointer-types + # this only occues on mingw + case $host in + *-*-mingw*) + CFLAGS_NODIST="$CFLAGS_NODIST -Wno-error=incompatible-pointer-types" + ;; + esac + PY_CHECK_CC_WARNING([enable], [extra], [if we can add -Wextra]) AS_VAR_IF([ac_cv_enable_extra_warning], [yes], [CFLAGS_NODIST="$CFLAGS_NODIST -Wextra"]) From d48b2d21436a93e2658475d3fc3c29708eb34cef Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 8 Sep 2024 15:04:42 +0530 Subject: [PATCH 115/123] test_makefile: normalize path so that it passes on mingw --- Lib/test/test_tools/test_makefile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index f1f0cd87ba7496..596e50dfbe2842 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -33,11 +33,12 @@ def list_test_dirs(self): if '\t' not in line: break result.append(line.replace('\\', '').strip()) + result = [os.path.normpath(d) for d in result if d] return result def test_makefile_test_folders(self): test_dirs = self.list_test_dirs() - idle_test = 'idlelib/idle_test' + idle_test = os.path.join('idlelib', 'idle_test') self.assertIn(idle_test, test_dirs) used = [idle_test] From 753581b306e259a2c9500e1989cbc04ab7875f02 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Thu, 12 Sep 2024 17:42:31 +0530 Subject: [PATCH 116/123] `sysconfig.get_platform()`: use consistent naming Currently, the platform names are hardcoded in many places, and are not named consistently. This commit fixes it by standardizing the names to be returned by `sysconfig.get_platform`. The naming is based on https://github.com/msys2-contrib/cpython-mingw/issues/167#issuecomment-1952215164 Similarly, `EXT_SUFFIX` is also standardized to be consistent with the platform names. Signed-off-by: Naveen M K --- Lib/site.py | 32 +++++++++------ Lib/sysconfig.py | 32 +++++++++------ Lib/test/test_importlib/test_windows.py | 38 +++++++++-------- configure.ac | 54 ++++++++++--------------- mingw_smoketests.py | 2 +- 5 files changed, 83 insertions(+), 75 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index c1ecd6ec4739c3..16c7c41cd3d0c6 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -314,21 +314,27 @@ def joinuser(*args): def _get_platform(): if os.name == 'nt': if 'gcc' in sys.version.lower(): + platform = 'mingw' + if 'amd64' in sys.version.lower(): + platform += '_x86_64' + elif 'arm64' in sys.version.lower(): + platform += '_aarch64' + elif 'arm' in sys.version.lower(): + platform += '_armv7' + else: + platform += '_i686' + if 'ucrt' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_ucrt' - return 'mingw_i686_ucrt' + platform += '_ucrt' + else: + platform += "_msvcrt" + if 'clang' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_clang' - if 'arm64' in sys.version.lower(): - return 'mingw_aarch64' - if 'arm' in sys.version.lower(): - return 'mingw_armv7' - return 'mingw_i686_clang' - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64' - return 'mingw_i686' + platform += "_llvm" + else: + platform += "_gnu" + + return platform return sys.platform # Same to sysconfig.get_path('purelib', os.name+'_user') diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 916c1e82b2d9b7..c12ce38dec5b3d 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -818,21 +818,27 @@ def get_platform(): """ if os.name == 'nt': if 'gcc' in sys.version.lower(): + platform = 'mingw' + if 'amd64' in sys.version.lower(): + platform += '_x86_64' + elif 'arm64' in sys.version.lower(): + platform += '_aarch64' + elif 'arm' in sys.version.lower(): + platform += '_armv7' + else: + platform += '_i686' + if 'ucrt' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_ucrt' - return 'mingw_i686_ucrt' + platform += '_ucrt' + else: + platform += "_msvcrt" + if 'clang' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_clang' - if 'arm64' in sys.version.lower(): - return 'mingw_aarch64' - if 'arm' in sys.version.lower(): - return 'mingw_armv7' - return 'mingw_i686_clang' - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64' - return 'mingw_i686' + platform += "_llvm" + else: + platform += "_gnu" + + return platform if 'amd64' in sys.version.lower(): return 'win-amd64' if '(arm)' in sys.version.lower(): diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index 03b5a98974f3fa..0b1254c2d75896 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -25,22 +25,28 @@ def get_platform(): 'arm' : 'win-arm32', } if os.name == 'nt': - if 'gcc' in sys.version.lower(): - if 'ucrt' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_ucrt' - return 'mingw_i686_ucrt' - if 'clang' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_clang' - if 'arm64' in sys.version.lower(): - return 'mingw_aarch64' - if 'arm' in sys.version.lower(): - return 'mingw_armv7' - return 'mingw_i686_clang' - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64' - return 'mingw_i686' + if "gcc" in sys.version.lower(): + platform = "mingw" + if "amd64" in sys.version.lower(): + platform += "_x86_64" + elif "arm64" in sys.version.lower(): + platform += "_aarch64" + elif "arm" in sys.version.lower(): + platform += "_armv7" + else: + platform += "_i686" + + if "ucrt" in sys.version.lower(): + platform += "_ucrt" + else: + platform += "_msvcrt" + + if "clang" in sys.version.lower(): + platform += "_llvm" + else: + platform += "_gnu" + + return platform if ('VSCMD_ARG_TGT_ARCH' in os.environ and os.environ['VSCMD_ARG_TGT_ARCH'] in TARGET_TO_PLAT): return TARGET_TO_PLAT[os.environ['VSCMD_ARG_TGT_ARCH']] diff --git a/configure.ac b/configure.ac index 5d1b6e25269823..264bae613ef66a 100644 --- a/configure.ac +++ b/configure.ac @@ -6335,16 +6335,12 @@ AC_C_BIGENDIAN AC_SUBST(PYD_PLATFORM_TAG) # Special case of PYD_PLATFORM_TAG with python build with mingw. -# Python can with compiled with clang or gcc and linked -# to msvcrt or ucrt. To avoid conflicts between them -# we are selecting the extension as based on the compiler -# and the runtime they link to -# gcc + x86_64 + msvcrt = cp{version number}-x86_64 -# gcc + i686 + msvcrt = cp{version number}-i686 -# gcc + x86_64 + ucrt = cp{version number}-x86_64-ucrt -# clang + x86_64 + ucrt = cp{version number}-x86_64-clang -# clang + i686 + ucrt = cp{version number}-i686-clang - +# Python can with different cpu arch and c runtime as well as different +# toolchain. We follow this`mingw___` +# convention for PYD_PLATFORM_TAG. Where: +# `cpu_arch` = `x86_64`, `aarch64` or `i686` +# `c_runtime` = `msvcrt` or `ucrt` +# `toolchain` = `gnu` or `llvm` PYD_PLATFORM_TAG="" case $host in *-*-mingw*) @@ -6363,38 +6359,32 @@ esac case $host_os in mingw*) AC_MSG_CHECKING(PYD_PLATFORM_TAG) + PYD_PLATFORM_TAG="mingw" case $host in i686-*-mingw*) - if test -n "${cc_is_clang}"; then - # it is CLANG32 - PYD_PLATFORM_TAG="mingw_i686_clang" - else - if test $linking_to_ucrt = no; then - PYD_PLATFORM_TAG="mingw_i686" - else - PYD_PLATFORM_TAG="mingw_i686_ucrt" - fi - fi + PYD_PLATFORM_TAG+="_i686" ;; x86_64-*-mingw*) - if test -n "${cc_is_clang}"; then - # it is CLANG64 - PYD_PLATFORM_TAG="mingw_x86_64_clang" - else - if test $linking_to_ucrt = no; then - PYD_PLATFORM_TAG="mingw_x86_64" - else - PYD_PLATFORM_TAG="mingw_x86_64_ucrt" - fi - fi + PYD_PLATFORM_TAG+="_x86_64" ;; aarch64-*-mingw*) - PYD_PLATFORM_TAG+="mingw_aarch64" + PYD_PLATFORM_TAG+="_aarch64" ;; armv7-*-mingw*) - PYD_PLATFORM_TAG+="mingw_armv7" + PYD_PLATFORM_TAG+="_armv7" ;; esac + if test $linking_to_ucrt = no; then + PYD_PLATFORM_TAG+="_msvcrt" + else + PYD_PLATFORM_TAG += "_ucrt" + fi + if test -n "${cc_is_clang}"; then + # it is CLANG32 + PYD_PLATFORM_TAG+="_llvm" + else + PYD_PLATFORM_TAG+="_gnu" + fi AC_MSG_RESULT($PYD_PLATFORM_TAG) esac diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 0ab5d3b60a443e..9ad32b3a126db9 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -37,7 +37,7 @@ if sysconfig.is_python_build(): os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" -_UCRT = sysconfig.get_platform() not in ('mingw_x86_64', 'mingw_i686') +_UCRT = 'ucrt' in sysconfig.get_platform() class Tests(unittest.TestCase): From ce5f345ca13b7c39e01e5d5beca1f260e7d591a1 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 1 Nov 2024 11:47:01 +0530 Subject: [PATCH 117/123] Add ucrt to version string --- Python/getcompiler.c | 5 +++++ configure.ac | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 4b0b9b328ba2f8..dd4d3c8f101468 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -32,8 +32,13 @@ #if defined(__clang__) #define str(x) #x #define xstr(x) str(x) +#if defined(_UCRT) +#define COMPILER COMP_SEP "[GCC UCRT Clang " xstr(__clang_major__) "." \ + xstr(__clang_minor__) "." xstr(__clang_patchlevel__) ARCH_SUFFIX "]" +#else #define COMPILER COMP_SEP "[GCC Clang " xstr(__clang_major__) "." \ xstr(__clang_minor__) "." xstr(__clang_patchlevel__) ARCH_SUFFIX "]" +#endif #else #if defined(_UCRT) #define COMPILER COMP_SEP "[GCC UCRT " __VERSION__ ARCH_SUFFIX "]" diff --git a/configure.ac b/configure.ac index 264bae613ef66a..10a6e8464f9add 100644 --- a/configure.ac +++ b/configure.ac @@ -6377,7 +6377,7 @@ case $host_os in if test $linking_to_ucrt = no; then PYD_PLATFORM_TAG+="_msvcrt" else - PYD_PLATFORM_TAG += "_ucrt" + PYD_PLATFORM_TAG+="_ucrt" fi if test -n "${cc_is_clang}"; then # it is CLANG32 From 33a869a80d7fc7dfc4df12d48104e7e56ce4a2aa Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sat, 2 Nov 2024 11:30:54 +0530 Subject: [PATCH 118/123] socketmodule: fix captilization of headers `Rpc` should be `rpc`. It error's out in cross-compilation. --- Modules/socketmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 7038e10d663e41..f0d84d350b6ea8 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -279,7 +279,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # endif /* Helpers needed for AF_HYPERV */ -# include +# include /* Macros based on the IPPROTO enum, see: https://bugs.python.org/issue29515 */ #ifdef _MSC_VER From 884f1500fd1aeef48bffc926d024370eb62ba511 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 22 Dec 2024 13:45:24 +0100 Subject: [PATCH 119/123] mingw_smoketests: build extension in a venv ensurepip no longer works since we implement PEP 668, so do everything in a venv. --- mingw_smoketests.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 9ad32b3a126db9..5af27645e5df7e 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -287,10 +287,14 @@ def test_c_ext_build(self): import sys import subprocess import textwrap + import venv from pathlib import Path with tempfile.TemporaryDirectory() as tmppro: - subprocess.check_call([sys.executable, "-m", "ensurepip", "--user"]) + builder = venv.EnvBuilder(with_pip=True) + builder.create(tmppro) + venv_exe = os.path.join(tmppro, "bin", os.path.basename(sys.executable)) + with Path(tmppro, "setup.py").open("w") as f: f.write( textwrap.dedent( @@ -342,20 +346,11 @@ def test_c_ext_build(self): ) ) subprocess.check_call( - [sys.executable, "-c", "import struct"], - ) - subprocess.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - "wheel", - ], + [venv_exe, "-c", "import struct"], ) subprocess.check_call( [ - sys.executable, + venv_exe, "-m", "pip", "install", @@ -363,7 +358,7 @@ def test_c_ext_build(self): ], ) subprocess.check_call( - [sys.executable, "-c", "import cwrapper"], + [venv_exe, "-c", "import cwrapper"], ) From 8f366d6aef7ac9c76c4c32c18633d8f2b68cf7bd Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Mon, 23 Dec 2024 06:03:11 +0100 Subject: [PATCH 120/123] venvlauncher: try looking for the versioned .exe first and then fall back By default venvlauncher only looks for python.exe in the python "home", which fails in the MSYS2 case where there could be multiple python versions in the same prefix i.e. python.exe could be 3.12 while the venv was created with python3.13.exe. Upstream CPython doesn't have this problem since they never have multiple Python versions in the same prefix. On the other hand the Python test suite (test_sysconfig specifically) creates a venv for the uninstalled Python build dir, where only python.exe exists. To cover both cases we first try to look for the python3.XY.exe and then fall back to python.exe. --- Makefile.pre.in | 4 ++-- PC/launcher.c | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index e9ded19a3fb519..68fbe7b1e7d9ec 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -827,10 +827,10 @@ $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) python_exe.o $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) python_exe.o $(BUILDVENVLAUNCHER): $(BUILDPYTHON) venvlauncher.o $(srcdir)/PC/launcher.c - $(LINKCC) -D_CONSOLE -DVENV_REDIRECT $(PY_STDMODULE_CFLAGS) -municode -static -static-libgcc -static-libstdc++ venvlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion + $(LINKCC) -D_CONSOLE -DVENV_REDIRECT -DPYTHON_EXECUTABLE_WITH_VERSION="L\"python$(LDVERSION)$(EXE)\"" $(PY_STDMODULE_CFLAGS) -municode -static -static-libgcc -static-libstdc++ venvlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion $(BUILDVENVWLAUNCHER): $(BUILDPYTHONW) venvwlauncher.o $(srcdir)/PC/launcher.c - $(LINKCC) -D_WINDOWS -DVENV_REDIRECT $(PY_STDMODULE_CFLAGS) -mwindows -municode -static -static-libgcc -static-libstdc++ venvwlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion + $(LINKCC) -D_WINDOWS -DVENV_REDIRECT -DPYTHON_EXECUTABLE_WITH_VERSION="L\"pythonw$(LDVERSION)$(EXE)\"" $(PY_STDMODULE_CFLAGS) -mwindows -municode -static -static-libgcc -static-libstdc++ venvwlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform diff --git a/PC/launcher.c b/PC/launcher.c index 14d8b8fff83d0c..ee7600468f1c8b 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1930,7 +1930,8 @@ process(int argc, wchar_t ** argv) if (!cch) { error(0, L"Cannot determine memory for home path"); } - cch += (DWORD)wcslen(PYTHON_EXECUTABLE) + 4; /* include sep, null and quotes */ + cch += (DWORD)max(wcslen(PYTHON_EXECUTABLE_WITH_VERSION), + wcslen(PYTHON_EXECUTABLE)) + 4; /* include sep, null and quotes */ executable = (wchar_t *)malloc(cch * sizeof(wchar_t)); if (executable == NULL) { error(RC_NO_MEMORY, L"A memory allocation failed"); @@ -1948,13 +1949,22 @@ process(int argc, wchar_t ** argv) executable[cch_actual++] = L'\\'; executable[cch_actual] = L'\0'; } - if (wcscat_s(&executable[1], cch - 1, PYTHON_EXECUTABLE)) { + if (wcscat_s(&executable[1], cch - 1, PYTHON_EXECUTABLE_WITH_VERSION)) { error(RC_BAD_VENV_CFG, L"Cannot create executable path from '%ls'", venv_cfg_path); } /* there's no trailing quote, so we only have to skip one character for the test */ + // Check if the versioned executable (PYTHON_EXECUTABLE_WITH_VERSION) exists first if (GetFileAttributesW(&executable[1]) == INVALID_FILE_ATTRIBUTES) { - error(RC_NO_PYTHON, L"No Python at '%ls'", executable); + // If not found, try PYTHON_EXECUTABLE + executable[cch_actual] = L'\0'; // Reset the path + if (wcscat_s(&executable[1], cch - 1, PYTHON_EXECUTABLE)) { + error(RC_BAD_VENV_CFG, L"Cannot create executable path from '%ls'", + venv_cfg_path); + } + if (GetFileAttributesW(&executable[1]) == INVALID_FILE_ATTRIBUTES) { + error(RC_NO_PYTHON, L"No Python at '%ls'", executable); + } } /* now append the final quote */ wcscat_s(executable, cch, L"\""); From 3fbe82e92e9c31702b50c09cadbc29c185014ec3 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 6 Feb 2025 11:12:20 +0100 Subject: [PATCH 121/123] CI: update actions --- .github/workflows/mingw.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 827dfe5a3705e5..65d7d7decfc524 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -22,7 +22,7 @@ jobs: run: | git config --global core.autocrlf false git config --global core.eol lf - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: msys2/setup-msys2@v2 with: msystem: ${{ matrix.msystem }} @@ -150,7 +150,7 @@ jobs: tar -zcf python.tar.gz python_pkgdir/ - name: Upload - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: build-${{ matrix.msystem }} @@ -161,7 +161,7 @@ jobs: container: image: ubuntu:24.10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install deps run: | apt-get update @@ -175,7 +175,7 @@ jobs: make \ pkg-config - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.12' @@ -207,7 +207,7 @@ jobs: zip -r install.zip _build/install - name: Upload - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: build-cross-gcc-x86_64 path: install.zip @@ -216,7 +216,7 @@ jobs: needs: [cross-gcc-x86_64] runs-on: windows-latest steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-cross-gcc-x86_64 @@ -235,7 +235,7 @@ jobs: matrix: arch: ['x86_64', 'i686', 'aarch64', 'armv7'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install deps run: | @@ -272,7 +272,7 @@ jobs: zip -r install.zip _build/install - name: Upload - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: build-cross-llvm-mingw-${{ matrix.arch }} path: install.zip @@ -285,7 +285,7 @@ jobs: matrix: arch: ['x86_64', 'i686'] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-cross-llvm-mingw-${{ matrix.arch }} From 4fd52baa016afc727b7465ea24373e79ea143b5d Mon Sep 17 00:00:00 2001 From: jeremyd2019 Date: Sun, 23 Jan 2022 16:38:18 -0800 Subject: [PATCH 122/123] CI: test CLANGARM64 test cross-compiled arm builds too --- .github/workflows/mingw.yml | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 65d7d7decfc524..97ecd188aa6d28 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -3,20 +3,27 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: windows-2022 + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - msystem: ['MINGW64','MINGW32','UCRT64','CLANG64'] + msystem: ['MINGW64','MINGW32','UCRT64','CLANG64','CLANGARM64'] include: - msystem: MINGW64 prefix: mingw-w64-x86_64 + runner: windows-2022 - msystem: MINGW32 prefix: mingw-w64-i686 + runner: windows-2022 - msystem: UCRT64 prefix: mingw-w64-ucrt-x86_64 + runner: windows-2022 - msystem: CLANG64 prefix: mingw-w64-clang-x86_64 + runner: windows-2022 + - msystem: CLANGARM64 + prefix: mingw-w64-clang-aarch64 + runner: windows-11-arm steps: - name: Setup git run: | @@ -26,7 +33,7 @@ jobs: - uses: msys2/setup-msys2@v2 with: msystem: ${{ matrix.msystem }} - release: false + release: ${{ matrix.msystem == 'CLANGARM64' }} update: true install: >- make @@ -279,11 +286,22 @@ jobs: cross-llvm-mingw-test: needs: [cross-llvm-mingw] - runs-on: windows-latest + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - arch: ['x86_64', 'i686'] + arch: ['x86_64', 'i686', 'aarch64', 'armv7'] + include: + - arch: x86_64 + runner: windows-latest + - arch: i686 + runner: windows-latest + - arch: aarch64 + runner: windows-11-arm + # we can test this for now, but Windows 11 24H2 dropped support for + # arm32 + - arch: armv7 + runner: windows-11-arm steps: - uses: actions/download-artifact@v4 with: @@ -291,7 +309,9 @@ jobs: - name: 'Run tests' run: | - 7z x install.zip + if (Test-Path ./_build) {rm -r -fo ./_build} + Add-Type -AssemblyName System.IO.Compression.FileSystem ; + [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\install.zip", "$PWD") ./_build/install/usr/local/bin/python3.exe -c "import sysconfig, pprint; pprint.pprint(sysconfig.get_config_vars())" From 5248fdecf1a5b7440848ab6705f211ac996d1c49 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 6 Jun 2025 21:32:42 +0200 Subject: [PATCH 123/123] CI: disable pkg-config to avoid it finding build machine packages --- .github/workflows/mingw.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 97ecd188aa6d28..360218a39f1439 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -197,6 +197,9 @@ jobs: mkdir _build && cd _build + export PKG_CONFIG_LIBDIR=/dev/null + unset PKG_CONFIG_PATH + ../configure \ --host=x86_64-w64-mingw32 \ --build=x86_64-pc-linux-gnu \ @@ -259,6 +262,9 @@ jobs: mkdir _build && cd _build + export PKG_CONFIG_LIBDIR=/dev/null + unset PKG_CONFIG_PATH + export CC=${{ matrix.arch }}-w64-mingw32-clang export CXX=${CC}++ ../configure \