From 3820da317a89366417e09b3d357e12badfde543f Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 17 Jun 2021 18:51:10 +0530 Subject: [PATCH 001/127] sysconfig: make _sysconfigdata.py relocatable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: lovetox <8661218+lovetox@users.noreply.github.com> --- Lib/sysconfig.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index acc8d4d1826b8a..7a7c0e20914e5b 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -2,6 +2,7 @@ import os import sys +import textwrap import threading from os.path import realpath @@ -524,11 +525,30 @@ def _generate_posix_vars(): os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') + replacement = """ + keys_to_replace = [ + 'BINDIR', 'BINLIBDEST', 'CONFINCLUDEDIR', + 'CONFINCLUDEPY', 'DESTDIRS', 'DESTLIB', 'DESTSHARED', + 'INCLDIRSTOMAKE', 'INCLUDEDIR', 'INCLUDEPY', + 'LIBDEST', 'LIBDIR', 'LIBPC', 'LIBPL', 'MACHDESTLIB', + 'MANDIR', 'SCRIPTDIR', 'datarootdir', 'exec_prefix', + 'TZPATH', + ] + + prefix = build_time_vars['BINDIR'][:-4] + + for key in keys_to_replace: + value = build_time_vars[key] + build_time_vars[key] = value.replace(prefix, sys.base_prefix) + """ + with open(destfile, 'w', encoding='utf8') as f: + f.write('import sys\n') f.write('# system configuration generated and used by' ' the sysconfig module\n') f.write('build_time_vars = ') pprint.pprint(vars, stream=f) + f.write('\n%s' % textwrap.dedent(replacement)) # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: From 2115fe6ff955e128126b6f7f30b9282489608a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:12 +0530 Subject: [PATCH 002/127] build: add --with-nt-threads and make it default on mingw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Naveen M K Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Include/internal/pycore_condvar.h | 10 ++ Include/pythread.h | 6 ++ Modules/_multiprocessing/multiprocessing.h | 5 +- configure.ac | 102 ++++++++++++++++++++- pyconfig.h.in | 3 + 5 files changed, 122 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 2177bed31fcd27..c46486f0947b8d 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -5,6 +5,12 @@ # error "this header requires Py_BUILD_CORE define" #endif +#ifdef __MINGW32__ +# if !defined(HAVE_PTHREAD_H) || defined(NT_THREADS) +# undef _POSIX_THREADS +# endif +#endif + #ifndef _POSIX_THREADS /* This means pthreads are not implemented in libc headers, hence the macro not present in unistd.h. But they still can be implemented as an external @@ -41,6 +47,10 @@ # define WIN32_LEAN_AND_MEAN #endif #include // CRITICAL_SECTION +/* winpthreads are involved via windows header, so need undef _POSIX_THREADS after header include */ +#if defined(_POSIX_THREADS) +#undef _POSIX_THREADS +#endif /* options */ /* non-emulated condition variables are provided for those that want diff --git a/Include/pythread.h b/Include/pythread.h index 63714437c496b7..5e84e3cbefc330 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -7,6 +7,12 @@ typedef void *PyThread_type_lock; extern "C" { #endif +#ifdef __MINGW32__ +# if !defined(HAVE_PTHREAD_H) || defined(NT_THREADS) +# undef _POSIX_THREADS +# endif +#endif + /* Return status codes for Python lock acquisition. Chosen for maximum * backwards compatibility, ie failure -> 0, success -> 1. */ typedef enum PyLockStatus { diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index dfc2a8e0799a60..dc92a233d374c6 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -23,7 +23,10 @@ # endif # define SEM_HANDLE HANDLE # define SEM_VALUE_MAX LONG_MAX -# define HAVE_MP_SEMAPHORE +# define HAVE_MP_SEMAPHORE +# if defined(HAVE_SEM_OPEN) && defined(_POSIX_THREADS) +# include +# endif #else # include /* O_CREAT and O_EXCL */ # if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) diff --git a/configure.ac b/configure.ac index 1a02d19f1b2a3f..de008a9be75b0e 100644 --- a/configure.ac +++ b/configure.ac @@ -2721,6 +2721,53 @@ then BASECFLAGS="$BASECFLAGS $ac_arch_flags" fi +dnl NOTE: +dnl - GCC 4.4+ for mingw* require and use posix threads(pthreads-w32) +dnl - Host may contain installed pthreads-w32. +dnl - On windows platform only NT-thread model is supported. +dnl To avoid miss detection scipt first will check for NT-thread model +dnl and if is not found will try to detect build options for pthread +dnl model. Autodetection could be overiden if variable with_nt_threads +dnl is set in "Site Configuration" (see autoconf manual). +dnl If NT-thread model is enabled script skips some checks that +dnl impact build process. When a new functionality is added, developers +dnl are responsible to update configure script to avoid thread models +dnl to be mixed. + +AC_MSG_CHECKING([for --with-nt-threads]) +AC_ARG_WITH(nt-threads, + AS_HELP_STRING([--with-nt-threads], [build with windows threads (default is system-dependent)]), +[ + case $withval in + no) with_nt_threads=no;; + yes) with_nt_threads=yes;; + *) with_nt_threads=yes;; + esac +], [ + case $host in + *-*-mingw*) with_nt_threads=yes;; + *) with_nt_threads=no;; + esac +]) +AC_MSG_RESULT([$with_nt_threads]) + +if test $with_nt_threads = yes ; then +AC_MSG_CHECKING([whether linking with nt-threads work]) +AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[]],[[_beginthread(0, 0, 0);]]) + ], + [AC_MSG_RESULT([yes])], + [AC_MSG_ERROR([failed to link with nt-threads])]) +fi + +if test $with_nt_threads = yes ; then + dnl temporary default flag to avoid additional pthread checks + dnl and initilize other ac..thread flags to no + ac_cv_pthread_is_default=no + ac_cv_kthread=no + ac_cv_pthread=no + dnl ac_cv_kpthread is set to no if default is yes (see below) +else # On some compilers, pthreads are available without further options # (e.g. MacOS X). On some of these systems, the compiler will not # complain if unaccepted options are passed (e.g. gcc on Mac OS X). @@ -2832,6 +2879,8 @@ int main(void){ CC="$ac_save_cc"]) fi +fi + # If we have set a CC compiler flag for thread support then # check if it works for CXX, too. if test ! -z "$CXX" @@ -2851,6 +2900,10 @@ elif test "$ac_cv_pthread" = "yes" then CXX="$CXX -pthread" ac_cv_cxx_thread=yes +elif test "$with_nt_threads" = "yes" +then + dnl set to always to skip extra pthread check below + ac_cv_cxx_thread=always else ac_cv_cxx_thread=no fi @@ -2892,8 +2945,8 @@ AC_CHECK_HEADERS([ \ alloca.h asm/types.h bluetooth.h conio.h crypt.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ ieeefp.h io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ linux/random.h linux/soundcard.h \ - linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ - sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ + linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pty.h \ + setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/pidfd.h sys/poll.h \ sys/random.h sys/resource.h sys/select.h sys/sendfile.h sys/socket.h sys/soundcard.h sys/stat.h \ @@ -2904,6 +2957,14 @@ AC_CHECK_HEADERS([ \ AC_HEADER_DIRENT AC_HEADER_MAJOR +# If using nt threads, don't look for pthread.h or thread.h +if test "x$with_nt_threads" = xno ; then +AC_HEADER_STDC +AC_CHECK_HEADERS(pthread.h sched.h thread.h) +AC_HEADER_DIRENT +AC_HEADER_MAJOR +fi + # bluetooth/bluetooth.h has been known to not compile with -std=c99. # http://permalink.gmane.org/gmane.linux.bluez.kernel/22294 SAVE_CFLAGS=$CFLAGS @@ -3116,6 +3177,10 @@ elif test "$ac_cv_pthread" = "yes" then CC="$CC -pthread" fi +if test $with_nt_threads = yes ; then + dnl skip check for pthread_t if NT-thread model is enabled + ac_cv_have_pthread_t=skip +else AC_CACHE_CHECK([for pthread_t], [ac_cv_have_pthread_t], [ AC_COMPILE_IFELSE([ AC_LANG_PROGRAM([[@%:@include ]], [[pthread_t x; x = *(pthread_t*)0;]]) @@ -3147,7 +3212,7 @@ AS_VAR_IF([ac_cv_pthread_key_t_is_arithmetic_type], [yes], [ AC_DEFINE([PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT], [1], [Define if pthread_key_t is compatible with int.]) ]) - +fi CC="$ac_save_cc" AC_MSG_CHECKING([for --enable-framework]) @@ -3679,10 +3744,15 @@ AS_VAR_IF([have_uuid], [missing], [ AS_VAR_IF([have_uuid], [missing], [have_uuid=no]) +if test $with_nt_threads = yes ; then + dnl do not search for sem_init if NT-thread model is enabled + : +else # 'Real Time' functions on Solaris # posix4 on Solaris 2.6 # pthread (first!) on Linux AC_SEARCH_LIBS([sem_init], [pthread rt posix4]) +fi # check if we need libintl for locale functions AC_CHECK_LIB([intl], [textdomain], @@ -4402,6 +4472,11 @@ then CXX="$CXX -pthread" fi posix_threads=yes +elif test $with_nt_threads = yes +then + posix_threads=no + AC_DEFINE(NT_THREADS, 1, + [Define to 1 if you want to use native NT threads]) else if test ! -z "$withval" -a -d "$withval" then LDFLAGS="$LDFLAGS -L$withval" @@ -4918,6 +4993,15 @@ else fi # checks for library functions +if test $with_nt_threads = yes ; then + dnl GCC(mingw) 4.4+ require and use posix threads(pthreads-w32) + dnl and host may contain installed pthreads-w32. + dnl Skip checks for some functions declared in pthreads-w32 if + dnl NT-thread model is enabled. + ac_cv_func_pthread_kill=skip + ac_cv_func_sem_open=skip + ac_cv_func_sched_setscheduler=skip +fi AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock close_range confstr \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ @@ -5814,6 +5898,10 @@ dnl actually works. For FreeBSD versions <= 7.2, dnl the kernel module that provides POSIX semaphores dnl isn't loaded by default, so an attempt to call dnl sem_open results in a 'Signal 12' error. +if test $with_nt_threads = yes ; then + dnl skip posix semaphores test if NT-thread model is enabled + ac_cv_posix_semaphores_enabled=no +fi AC_CACHE_CHECK([whether POSIX semaphores are enabled], [ac_cv_posix_semaphores_enabled], AC_RUN_IFELSE([ AC_LANG_SOURCE([ @@ -5847,6 +5935,14 @@ AS_VAR_IF([ac_cv_posix_semaphores_enabled], [no], [ ]) dnl Multiprocessing check for broken sem_getvalue +if test $with_nt_threads = yes ; then + dnl Skip test if NT-thread model is enabled. + dnl NOTE the test case below fail for pthreads-w32 as: + dnl - SEM_FAILED is not defined; + dnl - sem_open is a stub; + dnl - sem_getvalue work(!). + ac_cv_broken_sem_getvalue=skip +fi AC_CACHE_CHECK([for broken sem_getvalue], [ac_cv_broken_sem_getvalue], AC_RUN_IFELSE([ AC_LANG_SOURCE([ diff --git a/pyconfig.h.in b/pyconfig.h.in index df4d29fe5497a4..b359e13b5d9836 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1552,6 +1552,9 @@ /* Define if mvwdelch in curses.h is an expression. */ #undef MVWDELCH_IS_EXPRESSION +/* Define to 1 if you want to use native NT threads */ +#undef NT_THREADS + /* Define to the address where bug reports for this package should be sent. */ #undef PACKAGE_BUGREPORT From 0f488f1213d0c730a1d7a422c12da27141775ab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:13 +0530 Subject: [PATCH 003/127] Define `MS_WINDOWS` (and others) when compiling with MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is used throughout the Python code but is not defined when compiling with Mingw-w64 compilers, so define it. In MSVC these are defined in PC/pyconfig.h, we have to define it manually because pyconfig.h is created by autotools. Co-authored-by: Алексей Co-authored-by: Christoph Reiter Co-authored-by: Naveen M K --- Include/pyport.h | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Include/pyport.h b/Include/pyport.h index e2bac3bf504261..3d70b8e07815b1 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -53,6 +53,44 @@ #endif +#ifdef __MINGW32__ +/* Translate GCC[mingw*] platform specific defines to those + * used in python code. + */ +#if !defined(MS_WIN64) && defined(_WIN64) +# define MS_WIN64 +#endif +#if !defined(MS_WIN32) && defined(_WIN32) +# define MS_WIN32 +#endif +#if !defined(MS_WINDOWS) && defined(MS_WIN32) +# define MS_WINDOWS +#endif + +#if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN) || defined(Py_BUILD_CORE_MODULE) +#include + +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#define MS_WINDOWS_DESKTOP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +#define MS_WINDOWS_APP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_SYSTEM) +#define MS_WINDOWS_SYSTEM +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_GAMES) +#define MS_WINDOWS_GAMES +#endif + +/* Define to 1 if you support windows console io */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) +#define HAVE_WINDOWS_CONSOLE_IO 1 +#endif +#endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ + +#endif /* __MINGW32__*/ + /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. From ed0ed208960ca1a9c8bf57b5440adb66a8bac2a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:14 +0530 Subject: [PATCH 004/127] configure: add `MACHDEP` and `platform` on MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- configure.ac | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/configure.ac b/configure.ac index de008a9be75b0e..d003a531890741 100644 --- a/configure.ac +++ b/configure.ac @@ -550,6 +550,9 @@ then *-*-cygwin*) ac_sys_system=Cygwin ;; + *-*-mingw*) + ac_sys_system=MINGW + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -585,6 +588,7 @@ then linux*) MACHDEP="linux";; cygwin*) MACHDEP="cygwin";; darwin*) MACHDEP="darwin";; + mingw*) MACHDEP="win32";; '') MACHDEP="unknown";; esac @@ -618,6 +622,9 @@ if test "$cross_compiling" = yes; then ;; wasm32-*-* | wasm64-*-*) _host_cpu=$host_cpu + ;; + *-*-mingw*) + _host_cpu= ;; *) # for now, limit cross builds to known configurations @@ -625,6 +632,14 @@ if test "$cross_compiling" = yes; then AC_MSG_ERROR([cross build not supported for $host]) esac _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + + case "$host_os" in + mingw*) + # As sys.platform() return 'win32' to build python and extantions + # we will use 'mingw' (in setup.py and etc.) + _PYTHON_HOST_PLATFORM=mingw + ;; + esac fi # Some systems cannot stand _XOPEN_SOURCE being defined at all; they From 86cc1d25993c107dc08cb92d7eedaaae27624732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:15 +0530 Subject: [PATCH 005/127] Add default configuration for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Misc/config_mingw | 12 ++++++++++++ Misc/cross_mingw32 | 11 +++++++++++ configure.ac | 22 ++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 Misc/config_mingw create mode 100644 Misc/cross_mingw32 diff --git a/Misc/config_mingw b/Misc/config_mingw new file mode 100644 index 00000000000000..513065dec9b717 --- /dev/null +++ b/Misc/config_mingw @@ -0,0 +1,12 @@ +# configure defaults for mingw* hosts + +# mingw functions to ignore +ac_cv_func_ftruncate=ignore # implement it as _chsize + +# mingw-w64 functions to ignore +ac_cv_func_truncate=ignore +ac_cv_func_alarm=ignore + +# files to ignore +ac_cv_file__dev_ptmx=ignore #NOTE: under MSYS environment device exist +ac_cv_file__dev_ptc=no diff --git a/Misc/cross_mingw32 b/Misc/cross_mingw32 new file mode 100644 index 00000000000000..03fde9e6af3ad7 --- /dev/null +++ b/Misc/cross_mingw32 @@ -0,0 +1,11 @@ +# configure defaults for mingw32 host if cross-build + +ac_cv_little_endian_double=yes +ac_cv_big_endian_double=no +ac_cv_mixed_endian_double=no + +ac_cv_tanh_preserves_zero_sign=yes + +ac_cv_wchar_t_signed=no + +ac_cv_have_size_t_format=no diff --git a/configure.ac b/configure.ac index d003a531890741..8b11800daf52c4 100644 --- a/configure.ac +++ b/configure.ac @@ -1205,6 +1205,28 @@ AC_CACHE_CHECK([for -Wl,--no-as-needed], [ac_cv_wl_no_as_needed], [ ]) AC_SUBST([NO_AS_NEEDED]) +# initialize default configuration +py_config= +case $host in + *-*-mingw*) py_config=mingw ;; +esac +if test -n "$py_config" ; then + AC_MSG_NOTICE([loading configure defaults from .../Misc/config_$py_config"]) + . "$srcdir/Misc/config_$py_config" +fi + +# initialize defaults for cross-builds +if test "$cross_compiling" = yes; then + py_config=$host_os + case $py_config in + mingw32*) py_config=mingw32 ;; + esac + if test -f "$srcdir/Misc/cross_$py_config" ; then + AC_MSG_NOTICE([loading cross defaults from .../Misc/cross_$py_config"]) + . "$srcdir/Misc/cross_$py_config" + fi +fi + AC_MSG_CHECKING([for the Android API level]) cat > conftest.c < Date: Thu, 17 Jun 2021 18:51:16 +0530 Subject: [PATCH 006/127] configure: enable largefile support by default for Mingw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Largefile is supported on Windows. Co-authored-by: Naveen M K Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- configure.ac | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure.ac b/configure.ac index 8b11800daf52c4..6914b148382a72 100644 --- a/configure.ac +++ b/configure.ac @@ -3184,6 +3184,10 @@ dnl LFS does not work with Emscripten 3.1 AS_CASE([$ac_sys_system], [Emscripten], [have_largefile_support="no"] ) +dnl Activate on windows platforms (32&64-bit) where off_t(4) < fpos_t(8) +AS_CASE([$ac_sys_system], + [MINGW], [have_largefile_support="yes"] +) AS_VAR_IF([have_largefile_support], [yes], [ AC_DEFINE([HAVE_LARGEFILE_SUPPORT], [1], [Defined to enable large file support when an off_t is bigger than a long From 2c9f8d3885fbc3786086fbb39d4e3ef33c72b127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:18 +0530 Subject: [PATCH 007/127] Add `PC/` to `CPPFLAGS` and to `SRCDIRS` on Mingw Co-authored-by: Naveen M K --- configure.ac | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/configure.ac b/configure.ac index 6914b148382a72..aeb992878c22be 100644 --- a/configure.ac +++ b/configure.ac @@ -6913,6 +6913,16 @@ do THREADHEADERS="$THREADHEADERS \$(srcdir)/$h" done +case $host in + *-*-mingw*) + dnl Required for windows builds as Objects/exceptions.c require + dnl "errmap.h" from $srcdir/PC. + dnl Note we cannot use BASECPPFLAGS as autogenerated pyconfig.h + dnl has to be before customized located in ../PC. + CPPFLAGS="-I\$(srcdir)/PC $CPPFLAGS" + ;; +esac + AC_SUBST([SRCDIRS]) SRCDIRS="\ Modules \ @@ -6935,6 +6945,11 @@ SRCDIRS="\ Python \ Python/frozen_modules \ Python/deepfreeze" + +case $host in + *-*-mingw*) SRCDIRS="$SRCDIRS PC";; +esac + AC_MSG_CHECKING([for build directories]) for dir in $SRCDIRS; do if test ! -d $dir; then From 2976481933b3e7f0caf0ffeccac580f16f796ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:19 +0530 Subject: [PATCH 008/127] posixmodule.c: Define necessary headers for compiling on MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also, build `nt` module instead of `posix` when compiling with MINGW. Co-authored-by: Naveen M K Co-authored-by: Алексей --- Modules/Setup.bootstrap.in | 2 +- Modules/posixmodule.c | 28 ++++++++++++++++++++++++---- configure.ac | 8 ++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 8ef0f203a82a8e..1f21d3578ae358 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -8,7 +8,7 @@ # module C APIs are used in core atexit atexitmodule.c faulthandler faulthandler.c -posix posixmodule.c +@INITSYS@ posixmodule.c _signal signalmodule.c _tracemalloc _tracemalloc.c diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c21c6f06c729de..1a62ddd91d3b5e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -369,6 +369,26 @@ corresponding Unix manual entries for more information on calls."); # define HAVE_PIPE 1 # define HAVE_FSYNC 1 # define fsync _commit +#elif defined(__MINGW32__) /* GCC for windows hosts */ +/* getlogin is detected by configure on mingw-w64 */ +# undef HAVE_GETLOGIN +/*# define HAVE_GETCWD 1 - detected by configure*/ +# define HAVE_GETPPID 1 +# define HAVE_GETLOGIN 1 +# define HAVE_SPAWNV 1 +# define HAVE_WSPAWNV 1 +# define HAVE_WEXECV 1 +/*# define HAVE_EXECV 1 - detected by configure*/ +# define HAVE_PIPE 1 +# define HAVE_POPEN 1 +# define HAVE_SYSTEM 1 +# define HAVE_CWAIT 1 +# define HAVE_FSYNC 1 +# define fsync _commit +# include +# ifndef _MAX_ENV +# define _MAX_ENV 32767 +# endif #endif /* ! __WATCOMC__ || __QNX__ */ /*[clinic input] @@ -446,7 +466,7 @@ extern char *ctermid_r(char *); # endif #endif -#ifdef _MSC_VER +#ifdef MS_WINDOWS # ifdef HAVE_DIRECT_H # include # endif @@ -457,7 +477,7 @@ extern char *ctermid_r(char *); # include # endif # include -#endif /* _MSC_VER */ +#endif /* MS_WINDOWS */ #ifndef MAXPATHLEN # if defined(PATH_MAX) && PATH_MAX > 1024 @@ -1594,9 +1614,9 @@ _Py_Sigset_Converter(PyObject *obj, void *addr) ** man environ(7). */ #include -#elif !defined(_MSC_VER) && (!defined(__WATCOMC__) || defined(__QNX__) || defined(__VXWORKS__)) +#elif !defined(MS_WINDOWS) && (!defined(__WATCOMC__) || defined(__QNX__) || defined(__VXWORKS__)) extern char **environ; -#endif /* !_MSC_VER */ +#endif /* !MS_WINDOWS */ static PyObject * convertenviron(void) diff --git a/configure.ac b/configure.ac index aeb992878c22be..1bc49743c3d25a 100644 --- a/configure.ac +++ b/configure.ac @@ -755,6 +755,14 @@ then [Define to include mbstate_t for mbrtowc]) fi +AC_MSG_CHECKING([for init system calls]) +AC_SUBST(INITSYS) +case $host in + *-*-mingw*) INITSYS=nt;; + *) INITSYS=posix;; +esac +AC_MSG_RESULT([$INITSYS]) + # Record the configure-time value of MACOSX_DEPLOYMENT_TARGET, # it may influence the way we can build extensions, so distutils # needs to check it From 4ef3c6fcfc1828ee86c646438e5ab2fd52cabb64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:20 +0530 Subject: [PATCH 009/127] Build winreg by default when compiling for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter Co-authored-by: Naveen M K --- Modules/Setup.bootstrap.in | 3 +++ PC/winreg.c | 19 +++++++++++++++++++ configure.ac | 7 +++++++ 3 files changed, 29 insertions(+) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 1f21d3578ae358..3def7e8f810731 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -34,3 +34,6 @@ _symtable symtablemodule.c # for systems without $HOME env, used by site._getuserbase() @MODULE_PWD_TRUE@pwd pwdmodule.c + +# build-in modules for windows platform: +@USE_WIN32_MODULE@winreg ../PC/winreg.c diff --git a/PC/winreg.c b/PC/winreg.c index e2d5322f458c2a..368291d9dafa4e 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -19,6 +19,25 @@ #include "structmember.h" // PyMemberDef #include +#ifndef SIZEOF_HKEY +/* used only here */ +#if defined(MS_WIN64) +# define SIZEOF_HKEY 8 +#elif defined(MS_WIN32) +# define SIZEOF_HKEY 4 +#else +# error "SIZEOF_HKEY is not defined" +#endif +#endif + +#ifndef REG_LEGAL_CHANGE_FILTER +#define REG_LEGAL_CHANGE_FILTER (\ + REG_NOTIFY_CHANGE_NAME |\ + REG_NOTIFY_CHANGE_ATTRIBUTES |\ + REG_NOTIFY_CHANGE_LAST_SET |\ + REG_NOTIFY_CHANGE_SECURITY ) +#endif + #if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) typedef struct { diff --git a/configure.ac b/configure.ac index 1bc49743c3d25a..c2b1e6cb5c7323 100644 --- a/configure.ac +++ b/configure.ac @@ -4487,6 +4487,13 @@ done IFS=$as_save_IFS AC_MSG_RESULT([$DBM_CFLAGS $DBM_LIBS]) +# Determine if windows modules should be used. +AC_SUBST(USE_WIN32_MODULE) +USE_WIN32_MODULE='#' +case $host in + *-*-mingw*) USE_WIN32_MODULE=;; +esac + # Templates for things AC_DEFINEd more than once. # For a single AC_DEFINE, no template is needed. AH_TEMPLATE([_REENTRANT], From 6e8327cc3ab0167278f8ca0c69c5e9309f4275c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:23 +0530 Subject: [PATCH 010/127] pyport.h: Add support for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/pyport.h | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index 3d70b8e07815b1..ddff496ba80544 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -532,12 +532,12 @@ extern char * _getpty(int *, int, mode_t, int); */ /* - All windows ports, except cygwin, are handled in PC/pyconfig.h. + Only MSVC windows ports is handled in PC/pyconfig.h. - Cygwin is the only other autoconf platform requiring special + Cygwin and Mingw is the only other autoconf platform requiring special linkage handling and it uses __declspec(). */ -#if defined(__CYGWIN__) +#if defined(__CYGWIN__) || defined(__MINGW32__) # define HAVE_DECLSPEC_DLL #endif @@ -550,21 +550,23 @@ extern char * _getpty(int *, int, mode_t, int); # define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE # define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE /* module init functions inside the core need no external linkage */ - /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) + /* except for Cygwin/Mingw to handle embedding */ +# if defined(__CYGWIN__) || defined(__MINGW32__) # define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# else /* __CYGWIN__ */ +# else /* __CYGWIN__ || __MINGW32__*/ # define PyMODINIT_FUNC PyObject* -# endif /* __CYGWIN__ */ +# endif /* __CYGWIN__ || __MINGW32__*/ # else /* Py_BUILD_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ /* failures similar to those described at the bottom of 4.1: */ /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ -# if !defined(__CYGWIN__) +# if defined(__CYGWIN__) || defined(__MINGW32__) +# define PyAPI_FUNC(RTYPE) RTYPE +# else # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE -# endif /* !__CYGWIN__ */ +# endif /* __CYGWIN__ || __MINGW32__*/ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE /* module init functions outside the core must be exported */ # if defined(__cplusplus) From 0ff5f856a38d56035957951b1457715858c6b0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:24 +0530 Subject: [PATCH 011/127] configure: add options so that shared build is possible on MINGW Co-authored-by: Naveen M K --- configure.ac | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c2b1e6cb5c7323..d4e8b5f27558a0 100644 --- a/configure.ac +++ b/configure.ac @@ -1552,6 +1552,11 @@ if test $enable_shared = "yes"; then LDLIBRARY='libpython$(LDVERSION).dll.a' DLLLIBRARY='libpython$(LDVERSION).dll' ;; + MINGW*) + LDLIBRARY='libpython$(LDVERSION).dll.a' + DLLLIBRARY='libpython$(LDVERSION).dll' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + ;; SunOS*) LDLIBRARY='libpython$(LDVERSION).so' BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' @@ -1602,6 +1607,9 @@ else # shared is disabled BLDLIBRARY='$(LIBRARY)' LDLIBRARY='libpython$(LDVERSION).dll.a' ;; + MINGW*) + LDLIBRARY='libpython$(LDVERSION).a' + ;; esac fi @@ -3545,6 +3553,10 @@ then CYGWIN*) LDSHARED="gcc -shared -Wl,--enable-auto-image-base" LDCXXSHARED="g++ -shared -Wl,--enable-auto-image-base";; + MINGW*) + LDSHARED='$(CC) -shared -Wl,--enable-auto-image-base' + LDCXXSHARED='$(CXX) -shared -Wl,--enable-auto-image-base' + ;; *) LDSHARED="ld";; esac fi @@ -6934,7 +6946,8 @@ case $host in dnl "errmap.h" from $srcdir/PC. dnl Note we cannot use BASECPPFLAGS as autogenerated pyconfig.h dnl has to be before customized located in ../PC. - CPPFLAGS="-I\$(srcdir)/PC $CPPFLAGS" + dnl (-I. at end is workaround for setup.py logic) + CPPFLAGS="-I\$(srcdir)/PC $CPPFLAGS -I." ;; esac From e5719bec6bd661f18c1200b38997b5c5df39b521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:25 +0530 Subject: [PATCH 012/127] Add dynamic loading support for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Makefile.pre.in | 6 ++++++ Python/dynload_win.c | 9 +++++++-- configure.ac | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 7ca3dc62c01555..2d8145ff16508e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1378,6 +1378,12 @@ Python/dynload_hpux.o: $(srcdir)/Python/dynload_hpux.c Makefile -DSHLIB_EXT='"$(EXT_SUFFIX)"' \ -o $@ $(srcdir)/Python/dynload_hpux.c +Python/dynload_win.o: $(srcdir)/Python/dynload_win.c Makefile + $(CC) -c $(PY_CORE_CFLAGS) \ + -DSHLIB_SUFFIX='"$(SHLIB_SUFFIX)"' \ + -DEXT_SUFFIX='"$(EXT_SUFFIX)"' \ + -o $@ $(srcdir)/Python/dynload_win.c + Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h $(CC) -c $(PY_CORE_CFLAGS) \ -DABIFLAGS='"$(ABIFLAGS)"' \ diff --git a/Python/dynload_win.c b/Python/dynload_win.c index acab05e2c6def3..4bb90ba198b887 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -29,6 +29,12 @@ #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" const char *_PyImport_DynLoadFiletab[] = { +#ifdef EXT_SUFFIX + EXT_SUFFIX, /* include SOABI flags where is encoded debug */ +#endif +#ifdef SHLIB_SUFFIX + "-abi" PYTHON_ABI_STRING SHLIB_SUFFIX, +#endif PYD_TAGGED_SUFFIX, PYD_UNTAGGED_SUFFIX, NULL @@ -251,8 +257,7 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, ensure DLLs adjacent to the PYD are preferred. */ Py_BEGIN_ALLOW_THREADS hDLL = LoadLibraryExW(wpathname, NULL, - LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | - LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR); + LOAD_WITH_ALTERED_SEARCH_PATH); Py_END_ALLOW_THREADS PyMem_Free(wpathname); diff --git a/configure.ac b/configure.ac index d4e8b5f27558a0..1863fa8ed4f270 100644 --- a/configure.ac +++ b/configure.ac @@ -3424,6 +3424,9 @@ if test -z "$SHLIB_SUFFIX"; then CYGWIN*) SHLIB_SUFFIX=.dll;; *) SHLIB_SUFFIX=.so;; esac + case $host_os in + mingw*) SHLIB_SUFFIX=.dll;; + esac fi AC_MSG_RESULT([$SHLIB_SUFFIX]) @@ -5036,6 +5039,13 @@ then fi ;; esac + case $host in + *-*-mingw*) + DYNLOADFILE="dynload_win.o" + extra_machdep_objs="$extra_machdep_objs PC/dl_nt.o" + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"'" + ;; + esac fi AC_MSG_RESULT([$DYNLOADFILE]) if test "$DYNLOADFILE" != "dynload_stub.o" @@ -6925,6 +6935,12 @@ case "$ac_cv_computed_gotos" in yes*) AC_DEFINE([HAVE_COMPUTED_GOTOS], [1], [Define if the C compiler supports computed gotos.]) esac +case $host_os in + mingw*) + dnl Synchronized with _PyImport_DynLoadFiletab (dynload_win.c) + dnl Do not use more then one dot on this platform ! + EXT_SUFFIX=-$SOABI$SHLIB_SUFFIX;; +esac case $ac_sys_system in AIX*) From 69c4541a733021fda321c89f62888837df875ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:26 +0530 Subject: [PATCH 013/127] Ignore main program for frozen scripts on MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Makefile.pre.in | 2 +- configure.ac | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2d8145ff16508e..5e005b63e94597 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -383,7 +383,7 @@ PYTHON_OBJS= \ Python/errors.o \ Python/flowgraph.o \ Python/frame.o \ - Python/frozenmain.o \ + @PYTHON_OBJS_FROZENMAIN@ \ Python/future.o \ Python/getargs.o \ Python/getcompiler.o \ diff --git a/configure.ac b/configure.ac index 1863fa8ed4f270..e03f864b93b120 100644 --- a/configure.ac +++ b/configure.ac @@ -6967,6 +6967,16 @@ case $host in ;; esac +dnl Python interpreter main program for frozen scripts +AC_SUBST(PYTHON_OBJS_FROZENMAIN) +PYTHON_OBJS_FROZENMAIN="Python/frozenmain.o" +case $host in + *-*-mingw*) + dnl 'PC/frozen_dllmain.c' - not yet + PYTHON_OBJS_FROZENMAIN= + ;; +esac + AC_SUBST([SRCDIRS]) SRCDIRS="\ Modules \ From 477ada14d309018f034a77cc313d4012701cc87d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:30 +0530 Subject: [PATCH 014/127] Detect winsock2 and setup _socket module on MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Misc/config_mingw | 3 +++ Modules/socketmodule.c | 6 ++++++ configure.ac | 38 ++++++++++++++++++++++++++++++++------ pyconfig.h.in | 4 ++-- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/Misc/config_mingw b/Misc/config_mingw index 513065dec9b717..9be43fd3a11407 100644 --- a/Misc/config_mingw +++ b/Misc/config_mingw @@ -10,3 +10,6 @@ ac_cv_func_alarm=ignore # files to ignore ac_cv_file__dev_ptmx=ignore #NOTE: under MSYS environment device exist ac_cv_file__dev_ptc=no + +# force detection of winsock2 functionality - require wxp or newer +ac_cv_func_getpeername=yes diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 27afd73d9704db..7038e10d663e41 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -282,6 +282,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include /* Macros based on the IPPROTO enum, see: https://bugs.python.org/issue29515 */ +#ifdef _MSC_VER #define IPPROTO_ICMP IPPROTO_ICMP #define IPPROTO_IGMP IPPROTO_IGMP #define IPPROTO_GGP IPPROTO_GGP @@ -312,6 +313,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #define IPPROTO_PGM IPPROTO_PGM // WinSock2 only #define IPPROTO_L2TP IPPROTO_L2TP // WinSock2 only #define IPPROTO_SCTP IPPROTO_SCTP // WinSock2 only +#endif /* _MSC_VER */ /* Provides the IsWindows7SP1OrGreater() function */ #include @@ -426,6 +428,10 @@ remove_unusable_flags(PyObject *m) /* Do not include addrinfo.h for MSVC7 or greater. 'addrinfo' and * EAI_* constants are defined in (the already included) ws2tcpip.h. */ +#elif defined(__MINGW32__) + /* Do not include addrinfo.h as minimum supported version is + * _WIN32_WINNT >= WindowsXP(0x0501) + */ #else # include "addrinfo.h" #endif diff --git a/configure.ac b/configure.ac index e03f864b93b120..37554f7facf8ea 100644 --- a/configure.ac +++ b/configure.ac @@ -4188,6 +4188,12 @@ AS_CASE([$ac_sys_system], [OSSAUDIODEV_LIBS=""] ) +dnl On MINGW, you need to link against ws2_32 and iphlpapi for sockets to work +AS_CASE([$MACHDEP], + [win32], [SOCKET_LIBS="-lws2_32 -liphlpapi -lrpcrt4"], + [SOCKET_LIBS=""] +) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -5651,18 +5657,33 @@ if test $ac_cv_header_time_altzone = yes; then [Define this if your time.h defines altzone.]) fi +AC_CHECK_HEADERS([ws2tcpip.h]) AC_CACHE_CHECK([for addrinfo], [ac_cv_struct_addrinfo], -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], [[struct addrinfo a]])], +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +#ifdef HAVE_WS2TCPIP_H +# include +#else +# include +#endif]], + [[struct addrinfo a]])], [ac_cv_struct_addrinfo=yes], [ac_cv_struct_addrinfo=no])) if test $ac_cv_struct_addrinfo = yes; then - AC_DEFINE([HAVE_ADDRINFO], [1], [struct addrinfo (netdb.h)]) + AC_DEFINE([HAVE_ADDRINFO], [1], [struct addrinfo]) fi AC_CACHE_CHECK([for sockaddr_storage], [ac_cv_struct_sockaddr_storage], AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ -# include -@%:@ include ]], [[struct sockaddr_storage s]])], +#ifdef HAVE_WS2TCPIP_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif]], + [[struct sockaddr_storage s]])], [ac_cv_struct_sockaddr_storage=yes], [ac_cv_struct_sockaddr_storage=no])) if test $ac_cv_struct_sockaddr_storage = yes; then @@ -6855,8 +6876,11 @@ AC_CHECK_TYPE( [socklen_t], [], [AC_DEFINE( [socklen_t], [int], - [Define to `int' if does not define.] + [Define to `int' if or does not define.] )], [ +#ifdef HAVE_WS2TCPIP_H +#include +#endif #ifdef HAVE_SYS_TYPES_H #include #endif @@ -7665,7 +7689,9 @@ PY_STDLIB_MOD([mmap], PY_STDLIB_MOD([_socket], [], m4_flatten([test "$ac_cv_header_sys_socket_h" = "yes" -a "$ac_cv_header_sys_types_h" = "yes" - -a "$ac_cv_header_netinet_in_h" = "yes"])) + -a "$ac_cv_header_netinet_in_h" = "yes" + -o "$MACHDEP" = "win32"]), + [], [$SOCKET_LIBS]) dnl platform specific extensions PY_STDLIB_MOD([grp], [], [test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes]) diff --git a/pyconfig.h.in b/pyconfig.h.in index b359e13b5d9836..398b64f2f0be49 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -59,7 +59,7 @@ /* Define to 1 if you have the `acosh' function. */ #undef HAVE_ACOSH -/* struct addrinfo (netdb.h) */ +/* struct addrinfo */ #undef HAVE_ADDRINFO /* Define to 1 if you have the `alarm' function. */ @@ -1926,7 +1926,7 @@ /* Define to `unsigned int' if does not define. */ #undef size_t -/* Define to `int' if does not define. */ +/* Define to `int' if or does not define. */ #undef socklen_t /* Define to `int' if doesn't define. */ From d7e39c2227c85d0c66af905ede2ec320451feded Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 14 Feb 2023 20:40:42 +0530 Subject: [PATCH 015/127] Enable some windows specific modules --- Modules/Setup.bootstrap.in | 2 ++ Modules/Setup.stdlib.in | 7 +++++++ Modules/_winapi.c | 2 ++ PC/msvcrtmodule.c | 2 ++ Python/dynamic_annotations.c | 2 +- configure.ac | 10 ++++++++++ 6 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 3def7e8f810731..d8b41ee7f3b5e9 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -37,3 +37,5 @@ _symtable symtablemodule.c # build-in modules for windows platform: @USE_WIN32_MODULE@winreg ../PC/winreg.c +@MODULE_MSVCRT_TRUE@msvcrt -DPy_BUILD_CORE ../PC/msvcrtmodule.c +@MODULE__WINAPI_TRUE@_winapi _winapi.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 5e587dd3a5a397..98e3ef2fdf6a6f 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -160,6 +160,13 @@ # _scproxy needs SystemConfiguration and CoreFoundation framework @MODULE__SCPROXY_TRUE@_scproxy _scproxy.c +############################################################################ +# Windows specific modules + +@MODULE__OVERLAPPED_TRUE@_overlapped overlapped.c +@MODULE__MSI_TRUE@_msi ../PC/_msi.c +@MODULE_WINSOUND_TRUE@winsound ../PC/winsound.c + ############################################################################ # Test modules diff --git a/Modules/_winapi.c b/Modules/_winapi.c index ac3ee113ffda71..b5d14686d23c97 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -44,7 +44,9 @@ #endif #include "windows.h" #include +#if defined(Py_DEBUG) #include +#endif #include "winreparse.h" #if defined(MS_WIN32) && !defined(MS_WIN64) diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 53ef26b732f615..10cf78258a8084 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -22,7 +22,9 @@ #include #include #include +#ifdef _DEBUG #include +#endif #include #ifdef _MSC_VER diff --git a/Python/dynamic_annotations.c b/Python/dynamic_annotations.c index 7febaa09df1950..70d5b3dc72069c 100644 --- a/Python/dynamic_annotations.c +++ b/Python/dynamic_annotations.c @@ -27,7 +27,7 @@ * Author: Kostya Serebryany */ -#ifdef _MSC_VER +#ifdef MS_WINDOWS # include #endif diff --git a/configure.ac b/configure.ac index 37554f7facf8ea..8ccec5e9950f2f 100644 --- a/configure.ac +++ b/configure.ac @@ -7791,6 +7791,16 @@ PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) +dnl windows specific modules +PY_STDLIB_MOD([msvcrt], [test "$MACHDEP" = "win32"]) +PY_STDLIB_MOD([_winapi], [test "$MACHDEP" = "win32"]) +PY_STDLIB_MOD([_msi], [test "$MACHDEP" = "win32"], [], [], + [-lmsi -lcabinet -lrpcrt4]) +PY_STDLIB_MOD([winsound], [test "$MACHDEP" = "win32"], [], [], + [-lwinmm]) +PY_STDLIB_MOD([_overlapped], [test "$MACHDEP" = "win32"], [], [], + [-lws2_32]) + dnl test modules PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) From 407fcf09da49361208465a55b78f4a61233d2b61 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Tue, 21 Sep 2021 20:52:42 +0200 Subject: [PATCH 016/127] sysconfig: MINGW build extensions with GCC --- Lib/sysconfig.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 7a7c0e20914e5b..ed24aebe3a83c1 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -800,6 +800,8 @@ def get_platform(): """ if os.name == 'nt': + if 'GCC' in sys.version: + return 'mingw' if 'amd64' in sys.version.lower(): return 'win-amd64' if '(arm)' in sys.version.lower(): From d6665e5343d10e45decec51da0c8e052087a10f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:38 +0530 Subject: [PATCH 017/127] sysconfig: treat MINGW builds as POSIX builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Lib/sysconfig.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index ed24aebe3a83c1..e05a67d426cdd4 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -99,13 +99,18 @@ }, } +# GCC[mingw*] use posix build system +_POSIX_BUILD = os.name == 'posix' or \ + (os.name == "nt" and 'GCC' in sys.version) + # For the OS-native venv scheme, we essentially provide an alias: -if os.name == 'nt': +if os.name == 'nt' and not _POSIX_BUILD: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['nt_venv'] else: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] + # NOTE: site.py has copy of this function. # Sync it when modify this function. def _getuserbase(): @@ -120,7 +125,7 @@ def _getuserbase(): def joinuser(*args): return os.path.expanduser(os.path.join(*args)) - if os.name == "nt": + if os.name == "nt" and not _POSIX_BUILD: base = os.environ.get("APPDATA") or "~" return joinuser(base, "Python") @@ -288,7 +293,7 @@ def _expand_vars(scheme, vars): def _get_preferred_schemes(): - if os.name == 'nt': + if os.name == 'nt' and not _POSIX_BUILD: return { 'prefix': 'nt', 'home': 'posix_home', @@ -622,7 +627,7 @@ def parse_config_h(fp, vars=None): def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: - if os.name == "nt": + if os.name == "nt" and not _POSIX_BUILD: inc_dir = os.path.join(_PROJECT_BASE, "PC") else: inc_dir = _PROJECT_BASE @@ -690,10 +695,10 @@ def _init_config_vars(): except AttributeError: _CONFIG_VARS['py_version_nodot_plat'] = '' - if os.name == 'nt': + if os.name == 'nt' and not _POSIX_BUILD: _init_non_posix(_CONFIG_VARS) _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': + if _POSIX_BUILD: _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the @@ -703,7 +708,7 @@ def _init_config_vars(): # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) - if os.name == 'posix': + if _POSIX_BUILD: if _PYTHON_BUILD: # If srcdir is a relative path (typically '.' or '..') # then it should be interpreted relative to the directory From 2b1bb2c6dac4899eb9fc97c6914760c3fa413795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:39 +0530 Subject: [PATCH 018/127] Add support for stdcall without underscore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the case used in MINGW Co-authored-by: Алексей --- Modules/_ctypes/_ctypes.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index d3b496887b5692..c208223b3f14a5 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3416,6 +3416,18 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) mangled_name = alloca(strlen(name) + 1 + 1 + 1 + 3); /* \0 _ @ %d */ if (!mangled_name) return NULL; + /* Issue: for stdcall decorated export functions MSVC compiler adds + * underscore, but GCC compiler create them without. This is + * visible by example for _ctypes_test.pyd module. + * As well functions from system libraries are without underscore. + * Solutions: + * - If a python module is build with gcc option --add-stdcall-alias + * the module will contain XXX as alias for function XXX@ as result + * first search in this method will succeed. + * - Distutil may use compiler to create def-file, to modify it as + * add underscore alias and with new def file to create module. + * - Or may be just to search for function without underscore. + */ for (i = 0; i < 32; ++i) { sprintf(mangled_name, "_%s@%d", name, i*4); Py_BEGIN_ALLOW_THREADS @@ -3423,6 +3435,13 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) Py_END_ALLOW_THREADS if (address) return address; + /* search for function without underscore as weel */ + sprintf(mangled_name, "%s@%d", name, i*4); + Py_BEGIN_ALLOW_THREADS + address = (PPROC)GetProcAddress(handle, mangled_name); + Py_END_ALLOW_THREADS + if (address) + return address; } return NULL; #endif From 877eab86d815bb4e0ae5395aaa4933e77eb1ef51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:42 +0530 Subject: [PATCH 019/127] Customize site for MINGW MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Lib/site.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index aed254ad504d35..fadfccf9f77377 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -89,6 +89,12 @@ USER_BASE = None +# Same as defined in Lib/sysconfig.py +# redeclared since sysconfig is large for site. +# GCC[mingw*] use posix build system +_POSIX_BUILD = os.name == 'posix' or \ + (os.name == "nt" and 'GCC' in sys.version) + def _trace(message): if sys.flags.verbose: print(message, file=sys.stderr) @@ -294,7 +300,7 @@ def _getuserbase(): def joinuser(*args): return os.path.expanduser(os.path.join(*args)) - if os.name == "nt": + if os.name == "nt" and not _POSIX_BUILD: base = os.environ.get("APPDATA") or "~" return joinuser(base, "Python") @@ -309,7 +315,7 @@ def joinuser(*args): def _get_path(userbase): version = sys.version_info - if os.name == 'nt': + if os.name == 'nt' and not _POSIX_BUILD: ver_nodot = sys.winver.replace('.', '') return f'{userbase}\\Python{ver_nodot}\\site-packages' @@ -382,7 +388,7 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) - if os.sep == '/': + if _POSIX_BUILD: libdirs = [sys.platlibdir] if sys.platlibdir != "lib": libdirs.append("lib") From dcf5c715547953f7d1e2cae6b939e2b2880b580f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:43 +0530 Subject: [PATCH 020/127] add python config sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Misc/python-config.sh.in | 62 +++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 2602fe24c0402e..195c317ada3825 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -1,32 +1,44 @@ #!/bin/sh -# Keep this script in sync with python-config.in - exit_with_usage () { echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" - exit $1 + exit 1 } +# Really, python-config.py (and thus .sh) should be called directly, but +# sometimes software (e.g. GDB) calls python-config.sh as if it were the +# Python executable, passing python-config.py as the first argument. +# Work around that oddness by ignoring any .py passed as first arg. +case "$1" in + *.py) + shift + ;; +esac + if [ "$1" = "" ] ; then - exit_with_usage 1 + exit_with_usage fi # Returns the actual prefix where this script was installed to. installed_prefix () { - RESULT=$(dirname $(cd $(dirname "$1") && pwd -P)) - if which readlink >/dev/null 2>&1 ; then - if readlink -f "$RESULT" >/dev/null 2>&1; then - RESULT=$(readlink -f "$RESULT") - fi + local RESULT=$(dirname $(cd $(dirname "$1") && pwd -P)) + if [ $(which readlink) ] ; then + RESULT=$(readlink -f "$RESULT") + fi + # Since we don't know where the output from this script will end up + # we keep all paths in Windows-land since MSYS2 can handle that + # while native tools can't handle paths in MSYS2-land. + if [ "$OSTYPE" = "cygwin" ] || [ "$OSTYPE" = "msys" ] ; then + RESULT=$(cd "$RESULT" && pwd -W) fi echo $RESULT } prefix_real=$(installed_prefix "$0") -# Use sed to fix paths from their built-to locations to their installed-to +# Use sed to fix paths from their built-to locations to their installed to # locations. Keep prefix & exec_prefix using their original values in case # they are referenced in other configure variables, to prevent double # substitution, issue #22140. @@ -41,13 +53,17 @@ LIBM="@LIBM@" LIBC="@LIBC@" SYSLIBS="$LIBM $LIBC" ABIFLAGS="@ABIFLAGS@" +# Protect against lack of substitution. +if [ "$ABIFLAGS" = "@""ABIFLAGS""@" ] ; then + ABIFLAGS= +fi LIBS="@LIBPYTHON@ @LIBS@ $SYSLIBS" LIBS_EMBED="-lpython${VERSION}${ABIFLAGS} @LIBS@ $SYSLIBS" BASECFLAGS="@BASECFLAGS@" -LDLIBRARY="@LDLIBRARY@" OPT="@OPT@" PY_ENABLE_SHARED="@PY_ENABLE_SHARED@" LDVERSION="@LDVERSION@" +LDLIBRARY="@LDLIBRARY@" LIBDEST=${prefix_real}/lib/python${VERSION} LIBPL=$(echo "@LIBPL@" | sed "s#$prefix#$prefix_real#") SO="@EXT_SUFFIX@" @@ -61,7 +77,7 @@ for ARG in $* do case $ARG in --help) - exit_with_usage 0 + exit_with_usage ;; --embed) PY_EMBED=1 @@ -69,7 +85,7 @@ do --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--abiflags|--configdir) ;; *) - exit_with_usage 1 + exit_with_usage ;; esac done @@ -80,37 +96,37 @@ fi for ARG in "$@" do - case "$ARG" in + case $ARG in --prefix) - echo "$prefix_real" + echo -ne "$prefix_real" ;; --exec-prefix) - echo "$exec_prefix_real" + echo -ne "$exec_prefix_real " ;; --includes) - echo "$INCDIR $PLATINCDIR" + echo -ne "$INCDIR $PLATINCDIR" ;; --cflags) - echo "$INCDIR $PLATINCDIR $BASECFLAGS $CFLAGS $OPT" + echo -ne "$INCDIR $PLATINCDIR $BASECFLAGS $CFLAGS $OPT" ;; --libs) - echo "$LIBS" + echo -ne "$LIBS" ;; --ldflags) LIBPLUSED= if [ "$PY_ENABLE_SHARED" = "0" ] ; then LIBPLUSED="-L$LIBPL" fi - echo "$LIBPLUSED -L$libdir $LIBS" + echo -ne "$LIBPLUSED -L$libdir $LIBS " ;; --extension-suffix) - echo "$SO" + echo -ne "$SO " ;; --abiflags) - echo "$ABIFLAGS" + echo -ne "$ABIFLAGS " ;; --configdir) - echo "$LIBPL" + echo -ne "$LIBPL " ;; esac done From d11b445c7da07918151fee5d5141631ca8247ac8 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 17 Jun 2021 18:51:46 +0530 Subject: [PATCH 021/127] mingw: prefer unix sep if MSYSTEM environment variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter Co-authored-by: cat Co-authored-by: Naveen M K --- Include/pylifecycle.h | 6 ++ Lib/ntpath.py | 106 ++++++++++++++++++--------------- Modules/posixmodule.c | 2 + Python/initconfig.c | 2 +- Python/pathconfig.c | 135 ++++++++++++++++++++++++++++++++++++++++++ Python/traceback.c | 2 +- 6 files changed, 203 insertions(+), 50 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index e4c3b09c963fe8..50d8b76d74b610 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -21,6 +21,12 @@ PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); +PyAPI_FUNC(wchar_t) Py_GetSepW(const wchar_t *); +PyAPI_FUNC(char) Py_GetSepA(const char *); + +PyAPI_FUNC(void) Py_NormalizeSepsW(wchar_t *); +PyAPI_FUNC(void) Py_NormalizeSepsA(char *); + /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level * exit functions. diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 1bef630bd9c0ff..c4855ae11f36f0 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -11,9 +11,7 @@ curdir = '.' pardir = '..' extsep = '.' -sep = '\\' pathsep = ';' -altsep = '/' defpath = '.;C:\\bin' devnull = 'nul' @@ -23,6 +21,14 @@ import genericpath from genericpath import * +if sys.platform == "win32" and "MSYSTEM" in os.environ: + sep = '/' + altsep = '\\' +else: + sep = '\\' + altsep = '/' +bsep = str.encode(sep) +baltsep = str.encode(altsep) __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", @@ -35,9 +41,33 @@ def _get_bothseps(path): if isinstance(path, bytes): - return b'\\/' + return bsep+baltsep + else: + return sep+altsep + +def _get_sep(path): + if isinstance(path, bytes): + return bsep else: - return '\\/' + return sep + +def _get_altsep(path): + if isinstance(path, bytes): + return baltsep + else: + return altsep + +def _get_colon(path): + if isinstance(path, bytes): + return b':' + else: + return ':' + +def _get_unc_prefix(path): + if isinstance(path, bytes): + return b'\\\\?\\UNC\\' + else: + return '\\\\?\\UNC\\' # Normalize the case of a pathname and map slashes to backslashes. # Other normalizations (such as optimizing '../' away) are not done @@ -59,14 +89,14 @@ def normcase(s): return s if isinstance(s, bytes): encoding = sys.getfilesystemencoding() - s = s.decode(encoding, 'surrogateescape').replace('/', '\\') + s = s.decode(encoding, 'surrogateescape').replace(altsep, sep) s = _LCMapStringEx(_LOCALE_NAME_INVARIANT, _LCMAP_LOWERCASE, s) return s.encode(encoding, 'surrogateescape') else: return _LCMapStringEx(_LOCALE_NAME_INVARIANT, _LCMAP_LOWERCASE, - s.replace('/', '\\')) + s.replace(altsep, sep)) except ImportError: def normcase(s): """Normalize case of pathname. @@ -75,8 +105,8 @@ def normcase(s): """ s = os.fspath(s) if isinstance(s, bytes): - return os.fsencode(os.fsdecode(s).replace('/', '\\').lower()) - return s.replace('/', '\\').lower() + return os.fsencode(os.fsdecode(s).replace(altsep, sep).lower()) + return s.replace(altsep, sep).lower() # Return whether a path is absolute. @@ -88,14 +118,9 @@ def normcase(s): def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) - if isinstance(s, bytes): - sep = b'\\' - altsep = b'/' - colon_sep = b':\\' - else: - sep = '\\' - altsep = '/' - colon_sep = ':\\' + sep = _get_sep(s) + altsep = _get_altsep(s) + colon_sep = _get_colon(s) + sep s = s[:3].replace(altsep, sep) # Absolute: UNC, device, and paths with a drive and root. # LEGACY BUG: isabs("/x") should be false since the path has no drive. @@ -107,14 +132,9 @@ def isabs(s): # Join two (or more) paths. def join(path, *paths): path = os.fspath(path) - if isinstance(path, bytes): - sep = b'\\' - seps = b'\\/' - colon = b':' - else: - sep = '\\' - seps = '\\/' - colon = ':' + sep = _get_sep(path) + seps = _get_bothseps(path) + colon = _get_colon(path) try: if not paths: path[:0] + sep #23780: Ensure compatible data type even if p is null. @@ -189,18 +209,12 @@ def splitroot(p): splitroot('Windows/notepad') == ('', '', 'Windows/notepad') """ p = os.fspath(p) - if isinstance(p, bytes): - sep = b'\\' - altsep = b'/' - colon = b':' - unc_prefix = b'\\\\?\\UNC\\' - empty = b'' - else: - sep = '\\' - altsep = '/' - colon = ':' - unc_prefix = '\\\\?\\UNC\\' - empty = '' + sep = _get_sep(p) + altsep = _get_altsep(p) + colon = _get_colon(p) + unc_prefix = _get_unc_prefix(p) + empty = b'' if isinstance(p, bytes) else '' + normp = p.replace(altsep, sep) if normp[:1] == sep: if normp[1:2] == sep: @@ -258,9 +272,9 @@ def split(p): def splitext(p): p = os.fspath(p) if isinstance(p, bytes): - return genericpath._splitext(p, b'\\', b'/', b'.') + return genericpath._splitext(p, bsep, baltsep, b'.') else: - return genericpath._splitext(p, '\\', '/', '.') + return genericpath._splitext(p, sep, altsep, '.') splitext.__doc__ = genericpath._splitext.__doc__ @@ -528,14 +542,12 @@ def expandvars(path): def normpath(path): """Normalize path, eliminating double slashes, etc.""" path = os.fspath(path) + sep = _get_sep(path) + altsep = _get_altsep(path) if isinstance(path, bytes): - sep = b'\\' - altsep = b'/' curdir = b'.' pardir = b'..' else: - sep = '\\' - altsep = '/' curdir = '.' pardir = '..' path = path.replace(altsep, sep) @@ -773,6 +785,7 @@ def realpath(path, *, strict=False): # strip the prefix anyway. if ex.winerror == initial_winerror: path = spath + path = normpath(path) return path @@ -782,12 +795,11 @@ def realpath(path, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + sep = _get_sep(path) if isinstance(path, bytes): - sep = b'\\' curdir = b'.' pardir = b'..' else: - sep = '\\' curdir = '.' pardir = '..' @@ -842,13 +854,11 @@ def commonpath(paths): raise ValueError('commonpath() arg is an empty sequence') paths = tuple(map(os.fspath, paths)) + sep = _get_sep(paths[0]) + altsep = _get_altsep(paths[0]) if isinstance(paths[0], bytes): - sep = b'\\' - altsep = b'/' curdir = b'.' else: - sep = '\\' - altsep = '/' curdir = '.' try: diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 1a62ddd91d3b5e..2cb0c5bc6d361b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4006,6 +4006,7 @@ posix_getcwd(int use_bytes) return NULL; } + Py_NormalizeSepsW(wbuf2); PyObject *resobj = PyUnicode_FromWideChar(wbuf2, len); if (wbuf2 != wbuf) { PyMem_RawFree(wbuf2); @@ -4892,6 +4893,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) target_path = tmp; } + Py_NormalizeSepsW(target_path); result = PyUnicode_FromWideChar(target_path, result_length); if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); diff --git a/Python/initconfig.c b/Python/initconfig.c index ed4c4e23f34a80..6ba6f5bfc863b5 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -164,7 +164,7 @@ static const char usage_envvars[] = "PYTHONWARNINGS : warning control (-W)\n" ; -#if defined(MS_WINDOWS) +#if defined(_MSC_VER) # define PYTHONHOMEHELP "\\python{major}{minor}" #else # define PYTHONHOMEHELP "/lib/pythonX.X" diff --git a/Python/pathconfig.c b/Python/pathconfig.c index be0f97c4b204a9..3d684d68ff12df 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -18,6 +18,140 @@ extern "C" { #endif +#ifdef __MINGW32__ +#define wcstok wcstok_s +#include +#endif + +static int +Py_StartsWithA(const char * str, const char * prefix) +{ + while(*prefix) + { + if(*prefix++ != *str++) + return 0; + } + + return 1; +} + +static int +Py_StartsWithW(const wchar_t * str, const wchar_t * prefix) +{ + while(*prefix) + { + if(*prefix++ != *str++) + return 0; + } + + return 1; +} + +char +Py_GetSepA(const char *name) +{ + char* msystem = (char*)2; /* So that non Windows use / as sep */ + static char sep = '\0'; +#ifdef _WIN32 + /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx + * The "\\?\" prefix .. indicate that the path should be passed to the system with minimal + * modification, which means that you cannot use forward slashes to represent path separators + */ + if (name != NULL && Py_StartsWithA(name, "\\\\?\\") != 0) + { + return '\\'; + } +#endif + if (sep != '\0') + return sep; +#if defined(__MINGW32__) + msystem = Py_GETENV("MSYSTEM"); +#endif + if (msystem != NULL) + sep = '/'; + else + sep = '\\'; + return sep; +} + +static char +Py_GetAltSepA(const char *name) +{ + char sep = Py_GetSepA(name); + if (sep == '/') + return '\\'; + return '/'; +} + +void +Py_NormalizeSepsA(char *name) +{ + assert(name != NULL); + char sep = Py_GetSepA(name); + char altsep = Py_GetAltSepA(name); + char* seps; + if (name[0] != '\0' && name[1] == ':') { + name[0] = toupper(name[0]); + } + seps = strchr(name, altsep); + while(seps) { + *seps = sep; + seps = strchr(seps, altsep); + } +} + +wchar_t +Py_GetSepW(const wchar_t *name) +{ + char* msystem = (char*)2; /* So that non Windows use / as sep */ + static wchar_t sep = L'\0'; +#ifdef _WIN32 + /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx + * The "\\?\" prefix .. indicate that the path should be passed to the system with minimal + * modification, which means that you cannot use forward slashes to represent path separators + */ + if (name != NULL && Py_StartsWithW(name, L"\\\\?\\") != 0) + { + return L'\\'; + } +#endif + if (sep != L'\0') + return sep; +#if defined(__MINGW32__) + msystem = Py_GETENV("MSYSTEM"); +#endif + if (msystem != NULL) + sep = L'/'; + else + sep = L'\\'; + return sep; +} + +static wchar_t +Py_GetAltSepW(const wchar_t *name) +{ + char sep = Py_GetSepW(name); + if (sep == L'/') + return L'\\'; + return L'/'; +} + +void +Py_NormalizeSepsW(wchar_t *name) +{ + assert(name != NULL); + wchar_t sep = Py_GetSepW(name); + wchar_t altsep = Py_GetAltSepW(name); + wchar_t* seps; + if (name[0] != L'\0' && name[1] == L':') { + name[0] = towupper(name[0]); + } + seps = wcschr(name, altsep); + while(seps) { + *seps = sep; + seps = wcschr(seps, altsep); + } +} /* External interface */ @@ -317,6 +451,7 @@ _Py_SetProgramFullPath(const wchar_t *program_full_path) if (has_value && _Py_path_config.program_full_path == NULL) { path_out_of_memory(__func__); } + Py_NormalizeSepsW(_Py_path_config.program_name); } diff --git a/Python/traceback.c b/Python/traceback.c index 1cb82c57203758..37df982bb737f3 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -342,7 +342,7 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * filepath = PyBytes_AS_STRING(filebytes); /* Search tail of filename in sys.path before giving up */ - tail = strrchr(filepath, SEP); + tail = strrchr(filepath, Py_GetSepA(filepath)); if (tail == NULL) tail = filepath; else From 3c642b927b9f71071aff679f299a84f8803f19d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:50 +0530 Subject: [PATCH 022/127] msys cygwin semi native build sysconfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Lib/sysconfig.py | 8 +++++++ Makefile.pre.in | 7 ++++++ configure.ac | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index e05a67d426cdd4..60626d23b81365 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -451,6 +451,14 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): if isinstance(v, str): done[k] = v.strip() + # any keys that have one with the same name suffixed with _b2h + # need to be replaced with the value of the _b2h key. + # This converts from MSYS*/Cygwin paths to Windows paths. + for k, v in dict(done).items(): + if isinstance(k, str): + if k.endswith("_b2h"): + done[k[:-4]]=v + # save the results in the global dictionary vars.update(done) return vars diff --git a/Makefile.pre.in b/Makefile.pre.in index 5e005b63e94597..63ca7ad96a3f6c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -142,6 +142,13 @@ exec_prefix= @exec_prefix@ # Install prefix for data files datarootdir= @datarootdir@ +# Locations needed for semi-native fixup of sysconfig. +srcdir_b2h= @srcdir_b2h@ +VPATH_b2h= @VPATH_b2h@ +abs_srcdir_b2h= @abs_srcdir_b2h@ +abs_builddir_b2h= @abs_builddir_b2h@ +prefix_b2h= @prefix_b2h@ + # Expanded directories BINDIR= @bindir@ LIBDIR= @libdir@ diff --git a/configure.ac b/configure.ac index 8ccec5e9950f2f..c8407ad0472474 100644 --- a/configure.ac +++ b/configure.ac @@ -755,6 +755,65 @@ then [Define to include mbstate_t for mbrtowc]) fi +# On 'semi-native' build systems (MSYS*/Cygwin targeting MinGW-w64) +# _sysconfigdata.py will contain paths that are correct only in the +# build environment. This means external modules will fail to build +# without setting up the same env and also that the build of Python +# itself will fail as the paths are not correct for the host tools. +# +# Also, getpath.c uses GetModuleFileNameW (replacing \ with /) and +# compares that with the define VPATH (passed in via command-line) +# to determine whether it's the build- or the installed-Python. +# +# To work around these issues a set of _b2h variables are created: +# VPATH_b2h, prefix_b2h, srcdir_b2h, abs_srcdir_b2h +# and abs_builddir_b2h +# .. where b2h stands for build to host. sysconfig.py replaces path +# prefixes matching the non-b2h versions with the b2h equivalents. +# +# (note this assumes the host compilers are native and *not* cross +# - in the 'semi-native' scenario only that is.) + +AC_DEFUN([ABS_PATH_HOST], +[$1=$(cd $$2 && pwd) + case $build_os in + mingw*) + case $host_os in + mingw*) $1=$(cd $$2 && pwd -W) ;; + *) ;; + esac + ;; + cygwin*) + case $host_os in + mingw*) $1=$(cygpath -w -m $$2) ;; + *) ;; + esac + ;; + esac +AC_SUBST([$1]) +]) + +AC_MSG_CHECKING(absolute host location of VPATH) +ABS_PATH_HOST([VPATH_b2h],[srcdir]) +AC_MSG_RESULT([$VPATH_b2h]) + +AC_MSG_CHECKING(absolute host location of prefix) +ABS_PATH_HOST([prefix_b2h],[prefix]) +AC_MSG_RESULT([$prefix_b2h]) + +AC_MSG_CHECKING(absolute host location of srcdir) +ABS_PATH_HOST([srcdir_b2h],[srcdir]) +AC_MSG_RESULT([$srcdir_b2h]) + +AC_MSG_CHECKING(absolute host location of abs_srcdir) +ABS_PATH_HOST([abs_srcdir_b2h],[srcdir]) +AC_MSG_RESULT([$abs_srcdir_b2h]) + +my_builddir=. +AC_MSG_CHECKING(Absolute host location of abs_builddir) +ABS_PATH_HOST([abs_builddir_b2h],[my_builddir]) +AC_MSG_RESULT([$abs_builddir_b2h]) + AC_MSG_CHECKING([for init system calls]) AC_SUBST(INITSYS) case $host in From 6b8582b407c6da357e41cd85c34c8b1affa7631d Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Tue, 21 Sep 2021 20:53:59 +0200 Subject: [PATCH 023/127] sysconfig: mingw sysconfig like posix --- Lib/sysconfig.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 60626d23b81365..a0e07b93be6bf6 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -49,13 +49,13 @@ 'data': '{base}', }, 'nt': { - 'stdlib': '{installed_base}/Lib', - 'platstdlib': '{base}/Lib', - 'purelib': '{base}/Lib/site-packages', - 'platlib': '{base}/Lib/site-packages', - 'include': '{installed_base}/Include', - 'platinclude': '{installed_base}/Include', - 'scripts': '{base}/Scripts', + 'stdlib': '{installed_base}/lib/python{py_version_short}', + 'platstdlib': '{base}/lib/python{py_version_short}', + 'purelib': '{base}/lib/python{py_version_short}/site-packages', + 'platlib': '{base}/lib/python{py_version_short}/site-packages', + 'include': '{installed_base}/include/python{py_version_short}', + 'platinclude': '{installed_base}/include/python{py_version_short}', + 'scripts': '{base}/bin', 'data': '{base}', }, # Downstream distributors can overwrite the default install scheme. @@ -141,12 +141,12 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'stdlib': '{userbase}/lib/python{py_version_short}', + 'platstdlib': '{userbase}/lib/python{py_version_short}', + 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', + 'include': '{userbase}/include/python{py_version_short}', + 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'posix_user': { @@ -589,7 +589,7 @@ def _init_non_posix(vars): except IndexError: pass vars['EXE'] = '.exe' - vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT + vars['VERSION'] = _PY_VERSION_SHORT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) vars['TZPATH'] = '' From e6dd88b08f0491726146d4b4bfa3c1405aa91a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:53 +0530 Subject: [PATCH 024/127] mingw use backslashes in compileall py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Lib/compileall.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/compileall.py b/Lib/compileall.py index d394156cedc4e7..ef3633e9537615 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -38,6 +38,8 @@ def _walk_dir(dir, maxlevels, quiet=0): if name == '__pycache__': continue fullname = os.path.join(dir, name) + if sys.platform == "win32" and sys.version.find("GCC") >= 0: + fullname = fullname.replace('\\','/') if not os.path.isdir(fullname): yield fullname elif (maxlevels > 0 and name != os.curdir and name != os.pardir and From 1b0d7fda75368dcee7d5450672eb4ae718861d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:55 +0530 Subject: [PATCH 025/127] mingw pdcurses_ISPAD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/py_curses.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Include/py_curses.h b/Include/py_curses.h index e46b08e9cc414e..34228fbba55c81 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -36,6 +36,13 @@ #include #endif +#if defined(__MINGW32__) +#include +#if !defined(_ISPAD) +#define _ISPAD 0x10 +#endif +#endif + #ifdef HAVE_NCURSES_H /* configure was checking , but we will use , which has some or all these features. */ From c205e9912fd947734482c5cf850fb79d837f21e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:51:59 +0530 Subject: [PATCH 026/127] remove_path_max.default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/osdefs.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/osdefs.h b/Include/osdefs.h index 3243944a1483e9..99d49777a69d9a 100644 --- a/Include/osdefs.h +++ b/Include/osdefs.h @@ -10,7 +10,6 @@ extern "C" { #ifdef MS_WINDOWS #define SEP L'\\' #define ALTSEP L'/' -#define MAXPATHLEN 256 #define DELIM L';' #endif From 037e878c48bdaff130dbb6be551a4eee313cb5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:00 +0530 Subject: [PATCH 027/127] dont link with gettext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Modules/_localemodule.c | 7 +++++++ configure.ac | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index db8194372dae49..416bd591ea2664 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -12,6 +12,13 @@ This software comes with no warranty. Use at your own risk. #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_fileutils.h" +#ifdef __MINGW32__ +/* The header libintl.h and library libintl may exist on mingw host. + * To be compatible with MSVC build we has to undef some defines. + */ +#undef HAVE_LIBINTL_H +#undef HAVE_BIND_TEXTDOMAIN_CODESET +#endif #include #include diff --git a/configure.ac b/configure.ac index c8407ad0472474..cc7dc15d11f870 100644 --- a/configure.ac +++ b/configure.ac @@ -3878,10 +3878,19 @@ AC_SEARCH_LIBS([sem_init], [pthread rt posix4]) fi # check if we need libintl for locale functions +case $host in + *-*-mingw*) + dnl Native windows build don't use libintl (see _localemodule.c). + dnl Also we don't like setup.py to add "intl" library to the list + dnl when build _locale module. + ;; + *) AC_CHECK_LIB([intl], [textdomain], [AC_DEFINE([WITH_LIBINTL], [1], [Define to 1 if libintl is needed for locale functions.]) LIBS="-lintl $LIBS"]) + ;; +esac # checks for system dependent C++ extensions support case "$ac_sys_system" in From 893831ac31d5fccb8310e4f7e85cfaa49b9ba8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:01 +0530 Subject: [PATCH 028/127] ctypes python dll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Lib/ctypes/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 6cedee74236ea5..7fceeb8fa4e320 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -464,7 +464,9 @@ def LoadLibrary(self, name): cdll = LibraryLoader(CDLL) pydll = LibraryLoader(PyDLL) -if _os.name == "nt": +if _os.name == "nt" and _sys.version.find('GCC') >= 0: + pythonapi = PyDLL("libpython%d.%d%s.dll" % (_sys.version_info[:2] + (_sys.abiflags,)), None) +elif _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) From 269aa8cd9ad55e91228f305784d6645ad1b208a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:02 +0530 Subject: [PATCH 029/127] gdbm module includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Modules/_gdbmmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index ffd7b6420894b2..b536cedffce2a4 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -18,7 +18,7 @@ #include #include -#if defined(WIN32) && !defined(__CYGWIN__) +#if defined(WIN32) && !defined(__CYGWIN__) && !defined(__MINGW32__) #include "gdbmerrno.h" extern const char * gdbm_strerror(gdbm_error); #endif From cb41ac70252f2ef89f10203e7309f5add9eb6ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:03 +0530 Subject: [PATCH 030/127] use gnu_printf in format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/bytesobject.h | 4 ++-- Include/pyerrors.h | 4 ++-- Include/pyport.h | 6 ++++++ Include/sysmodule.h | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index ee448cd02bdab3..d10ec3f86cb696 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -35,9 +35,9 @@ PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyBytes_FromString(const char *); PyAPI_FUNC(PyObject *) PyBytes_FromObject(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list) - Py_GCC_ATTRIBUTE((format(printf, 1, 0))); + Py_PRINTF(1, 0); PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_PRINTF(1, 2); PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *); PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int); diff --git a/Include/pyerrors.h b/Include/pyerrors.h index d089fa71779330..f8e604eef546e8 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -321,9 +321,9 @@ PyAPI_FUNC(int) PyUnicodeTranslateError_SetReason( ); PyAPI_FUNC(int) PyOS_snprintf(char *str, size_t size, const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 3, 4))); + Py_PRINTF(3, 4); PyAPI_FUNC(int) PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) - Py_GCC_ATTRIBUTE((format(printf, 3, 0))); + Py_PRINTF(3, 0); #ifndef Py_LIMITED_API # define Py_CPYTHON_ERRORS_H diff --git a/Include/pyport.h b/Include/pyport.h index ddff496ba80544..df363d0daa2dbd 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -666,6 +666,12 @@ extern char * _getpty(int *, int, mode_t, int); #define Py_VA_COPY va_copy +#if defined(__GNUC__) && ((__GNUC__ == 4 && __GNUC_MINOR__>= 4) || __GNUC__ > 4) +# define Py_PRINTF(X,Y) Py_GCC_ATTRIBUTE((format(gnu_printf,X,Y))) +#else +# define Py_PRINTF(X,Y) Py_GCC_ATTRIBUTE((format(printf,X,Y))) +#endif + /* * Convenient macros to deal with endianness of the platform. WORDS_BIGENDIAN is * detected by configure and defined in pyconfig.h. The code in pyconfig.h diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 96f883870b34dc..55c558a9037b8e 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -15,9 +15,9 @@ Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int); Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetPath(const wchar_t *); PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_PRINTF(1, 2); PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_PRINTF(1, 2); PyAPI_FUNC(void) PySys_FormatStdout(const char *format, ...); PyAPI_FUNC(void) PySys_FormatStderr(const char *format, ...); From ac698395e3c6cc48ae6a4615101cabcb419ea8e5 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 17 Jun 2021 18:52:03 +0530 Subject: [PATCH 031/127] mingw fix ssl dont use enum_certificates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter --- Lib/ssl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index 42ebb8ed38466d..791d5c548abe9c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -254,7 +254,7 @@ class _TLSMessageType: CHANGE_CIPHER_SPEC = 0x0101 -if sys.platform == "win32": +if sys.platform == "win32" and sys.version.find("GCC") == -1: from _ssl import enum_certificates, enum_crls from socket import socket, SOCK_STREAM, create_connection @@ -528,7 +528,7 @@ def _load_windows_store_certs(self, storename, purpose): def load_default_certs(self, purpose=Purpose.SERVER_AUTH): if not isinstance(purpose, _ASN1Object): raise TypeError(purpose) - if sys.platform == "win32": + if sys.platform == "win32" and sys.version.find("GCC") == -1: for storename in self._windows_cert_stores: self._load_windows_store_certs(storename, purpose) self.set_default_verify_paths() From dbf3d4a853e6c0b6ebec51d9ac104c30ee718d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:06 +0530 Subject: [PATCH 032/127] fix using dllhandle and winver mingw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Python/sysmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6ed28ec5fbd8ba..0182569172b9dd 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -45,7 +45,7 @@ Data members: # include #endif /* MS_WINDOWS */ -#ifdef MS_COREDLL +#if defined(MS_WINDOWS) && defined(Py_ENABLE_SHARED) extern void *PyWin_DLLhModule; /* A string loaded from the DLL at startup: */ extern const char *PyWin_DLLVersionString; @@ -3429,7 +3429,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS_FROM_STRING("byteorder", "little"); #endif -#ifdef MS_COREDLL +#if defined(MS_WINDOWS) && defined(Py_ENABLE_SHARED) SET_SYS("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule)); SET_SYS_FROM_STRING("winver", PyWin_DLLVersionString); #endif From 6614b332854b172805467c0c02474e9f3210d0e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:07 +0530 Subject: [PATCH 033/127] Add AMD64 to sys config so msvccompiler get_build_version works --- Python/getcompiler.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Python/getcompiler.c b/Python/getcompiler.c index a5d26239e8772e..3e31c80bdf4fb1 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -10,7 +10,22 @@ #if defined(__clang__) #define COMPILER "[Clang " __clang_version__ "]" #elif defined(__GNUC__) -#define COMPILER "[GCC " __VERSION__ "]" +/* To not break compatibility with things that determine + CPU arch by calling get_build_version in msvccompiler.py + (such as NumPy) add "32 bit" or "64 bit (AMD64)" on Windows + and also use a space as a separator rather than a newline. */ +#if defined(_WIN32) +#define COMP_SEP " " +#if defined(__x86_64__) +#define ARCH_SUFFIX " 64 bit (AMD64)" +#else +#define ARCH_SUFFIX " 32 bit" +#endif +#else +#define COMP_SEP "\n" +#define ARCH_SUFFIX "" +#endif +#define COMPILER COMP_SEP "[GCC " __VERSION__ ARCH_SUFFIX "]" // Generic fallbacks. #elif defined(__cplusplus) #define COMPILER "[C++]" From e20536fb7ac99af6713301d0d3ccb2f7740fa6d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:08 +0530 Subject: [PATCH 034/127] MINGW link with additional library --- configure.ac | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configure.ac b/configure.ac index cc7dc15d11f870..7002b78672169a 100644 --- a/configure.ac +++ b/configure.ac @@ -7104,6 +7104,15 @@ for dir in $SRCDIRS; do done AC_MSG_RESULT([done]) +# For mingw build need additional library for linking +case $host in + *-*-mingw*) + LIBS="$LIBS -lversion -lshlwapi -lpathcch" + ;; + *) + ;; +esac + # Availability of -O2: AC_CACHE_CHECK([for -O2], [ac_cv_compile_o2], [ saved_cflags="$CFLAGS" From 47a51bd876d1e9b8fbeeb633211d0f16c0980a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C3=BAl=20Ibarra=20Corretg=C3=A9?= Date: Thu, 17 Jun 2021 18:52:09 +0530 Subject: [PATCH 035/127] install msilib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Christoph Reiter Co-authored-by: Ray Donnelly --- Makefile.pre.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 63ca7ad96a3f6c..5cc3d6dccae9ee 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2122,6 +2122,7 @@ LIBSUBDIRS= asyncio \ tkinter \ tomllib \ turtledemo \ + msilib \ unittest \ urllib \ venv venv/scripts venv/scripts/common venv/scripts/posix \ From 272a424a36f574ac03f181c8cfdb005dedf37812 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:10 +0530 Subject: [PATCH 036/127] fix signal module build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Modules/Setup.bootstrap.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index d8b41ee7f3b5e9..ec0f81eea2f4ff 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -9,7 +9,7 @@ atexit atexitmodule.c faulthandler faulthandler.c @INITSYS@ posixmodule.c -_signal signalmodule.c +_signal signalmodule.c -lws2_32 _tracemalloc _tracemalloc.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig From 3a96d1d80a52f6912ba0469278298d8bbc8bb7c5 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:11 +0530 Subject: [PATCH 037/127] build: build winconsoleio and _testconsole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Modules/Setup.bootstrap.in | 2 +- Modules/Setup.stdlib.in | 1 + configure.ac | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index ec0f81eea2f4ff..cec80f5a9f0f19 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -16,7 +16,7 @@ _tracemalloc _tracemalloc.c _codecs _codecsmodule.c _collections _collectionsmodule.c errno errnomodule.c -_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c +_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c _io/winconsoleio.c itertools itertoolsmodule.c _sre _sre/sre.c _thread _threadmodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 98e3ef2fdf6a6f..b96db5d6d2b1e9 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -184,6 +184,7 @@ @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c @MODULE__TESTMULTIPHASE_TRUE@_testsinglephase _testsinglephase.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c +@MODULE__TESTCONSOLE_TRUE@_testconsole ../PC/_testconsole.c # Limited API template modules; must be built as shared modules. @MODULE_XXLIMITED_TRUE@xxlimited xxlimited.c diff --git a/configure.ac b/configure.ac index 7002b78672169a..1c8d1e73a7289e 100644 --- a/configure.ac +++ b/configure.ac @@ -7887,6 +7887,7 @@ PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testconsole], [test "$TEST_MODULES" = yes -a "$MACHDEP" = "win32"]) PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes], [], [$LIBM]) From bc6d357a914d785bd85eefca2d870103df36ee55 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:12 +0530 Subject: [PATCH 038/127] expose sem_unlink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Modules/_multiprocessing/multiprocessing.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 8f9daa5c3de0cc..eb2cb5f86f2c9b 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -180,7 +180,7 @@ static PyMethodDef module_methods[] = { _MULTIPROCESSING_RECV_METHODDEF _MULTIPROCESSING_SEND_METHODDEF #endif -#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) +#if defined(MS_WINDOWS) || (!defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__)) _MULTIPROCESSING_SEM_UNLINK_METHODDEF #endif {NULL} From 0f752f55442fcdfaaf2f0fa66794ba09c9cd8625 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:14 +0530 Subject: [PATCH 039/127] Use cygpty while using isatty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit so that msys shells are detected as interactive ones Co-authored-by: Алексей Co-authored-by: Naveen M K --- Include/iscygpty.h | 41 ++++++++++ Makefile.pre.in | 2 + Modules/_io/fileio.c | 3 +- Modules/main.c | 3 +- Modules/posixmodule.c | 3 +- Objects/fileobject.c | 3 +- Python/bltinmodule.c | 5 +- Python/fileutils.c | 7 +- Python/frozenmain.c | 3 +- Python/iscygpty.c | 185 ++++++++++++++++++++++++++++++++++++++++++ Python/pylifecycle.c | 5 +- 11 files changed, 248 insertions(+), 12 deletions(-) create mode 100644 Include/iscygpty.h create mode 100644 Python/iscygpty.c diff --git a/Include/iscygpty.h b/Include/iscygpty.h new file mode 100644 index 00000000000000..82fd0affbd4875 --- /dev/null +++ b/Include/iscygpty.h @@ -0,0 +1,41 @@ +/* + * iscygpty.h -- part of ptycheck + * https://github.com/k-takata/ptycheck + * + * Copyright (c) 2015-2017 K.Takata + * + * You can redistribute it and/or modify it under the terms of either + * the MIT license (as described below) or the Vim license. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _ISCYGPTY_H +#define _ISCYGPTY_H + +#ifdef _WIN32 +int is_cygpty(int fd); +int is_cygpty_used(void); +#else +#define is_cygpty(fd) 0 +#define is_cygpty_used() 0 +#endif + +#endif /* _ISCYGPTY_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 5cc3d6dccae9ee..2af2beea09ecc7 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -406,6 +406,7 @@ PYTHON_OBJS= \ Python/instrumentation.o \ Python/intrinsics.o \ Python/legacy_tracing.o \ + Python/iscygpty.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ @@ -1636,6 +1637,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/import.h \ $(srcdir)/Include/interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ + $(srcdir)/Include/iscygpty.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ $(srcdir)/Include/longobject.h \ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index bab68077a2144a..919b8b0429e84b 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -20,6 +20,7 @@ #endif #include /* For offsetof */ #include "_iomodule.h" +#include "iscygpty.h" /* * Known likely problems: @@ -1143,7 +1144,7 @@ _io_FileIO_isatty_impl(fileio *self) return err_closed(); Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - res = isatty(self->fd); + res = isatty(self->fd) || is_cygpty(self->fd); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS return PyBool_FromLong(res); diff --git a/Modules/main.c b/Modules/main.c index f4e4018212f59e..fbdccddaee955c 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -7,6 +7,7 @@ #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "iscygpty.h" /* Includes for exit_sigint() */ #include // perror() @@ -92,7 +93,7 @@ static inline int config_run_code(const PyConfig *config) static int stdin_is_interactive(const PyConfig *config) { - return (isatty(fileno(stdin)) || config->interactive); + return (isatty(fileno(stdin)) || config->interactive || is_cygpty(fileno(stdin))); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2cb0c5bc6d361b..51b3e525403039 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -54,6 +54,7 @@ #ifdef __ANDROID__ # undef HAVE_FACCESSAT #endif +#include "iscygpty.h" #include // ctermid() #include // system() @@ -11235,7 +11236,7 @@ os_isatty_impl(PyObject *module, int fd) int return_value; Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - return_value = isatty(fd); + return_value = isatty(fd) || is_cygpty(fd); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS return return_value; diff --git a/Objects/fileobject.c b/Objects/fileobject.c index e99e155f2b8c98..c74d2af4903a55 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -3,6 +3,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "iscygpty.h" #include "pycore_runtime.h" // _PyRuntime #if defined(HAVE_GETC_UNLOCKED) && !defined(_Py_MEMORY_SANITIZER) @@ -382,7 +383,7 @@ stdprinter_isatty(PyStdPrinter_Object *self, PyObject *Py_UNUSED(ignored)) } Py_BEGIN_ALLOW_THREADS - res = isatty(self->fd); + res = isatty(self->fd) || is_cygpty(self->fd); Py_END_ALLOW_THREADS return PyBool_FromLong(res); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2ae28693bee05c..d918964a90e4b1 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1,6 +1,7 @@ /* Built-in functions */ #include "Python.h" +#include "iscygpty.h" #include #include "pycore_ast.h" // _PyAST_Validate() #include "pycore_call.h" // _PyObject_CallNoArgs() @@ -2163,7 +2164,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) if (fd < 0 && PyErr_Occurred()) { goto error; } - tty = fd == fileno(stdin) && isatty(fd); + tty = fd == fileno(stdin) && (isatty(fd) || is_cygpty(fd)); } if (tty) { tmp = PyObject_CallMethodNoArgs(fout, &_Py_ID(fileno)); @@ -2176,7 +2177,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) Py_DECREF(tmp); if (fd < 0 && PyErr_Occurred()) goto error; - tty = fd == fileno(stdout) && isatty(fd); + tty = fd == fileno(stdout) && (isatty(fd) || is_cygpty(fd)); } } diff --git a/Python/fileutils.c b/Python/fileutils.c index c7521751cd26d1..6d3b04ff5bb9e6 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "iscygpty.h" #include "pycore_fileutils.h" // fileutils definitions #include "pycore_runtime.h" // _PyRuntime #include "osdefs.h" // SEP @@ -77,7 +78,7 @@ _Py_device_encoding(int fd) int valid; Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - valid = isatty(fd); + valid = isatty(fd) || is_cygpty(fd); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS if (!valid) @@ -1925,12 +1926,12 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) depending on heap usage). */ if (gil_held) { Py_BEGIN_ALLOW_THREADS - if (isatty(fd)) { + if (isatty(fd) || is_cygpty(fd)) { count = 32767; } Py_END_ALLOW_THREADS } else { - if (isatty(fd)) { + if (isatty(fd) || is_cygpty(fd)) { count = 32767; } } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index f8be165f7671df..d18c7b3884c27b 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_runtime.h" // _PyRuntime_Initialize() #include +#include "iscygpty.h" #ifdef MS_WINDOWS extern void PyWinFreeze_ExeInit(void); @@ -71,7 +72,7 @@ Py_FrozenMain(int argc, char **argv) sts = 0; } - if (inspect && isatty((int)fileno(stdin))) { + if (inspect && (isatty((int)fileno(stdin)) || is_cygpty((int)fileno(stdin)))) sts = PyRun_AnyFile(stdin, "") != 0; } diff --git a/Python/iscygpty.c b/Python/iscygpty.c new file mode 100644 index 00000000000000..722f88f2f46dcb --- /dev/null +++ b/Python/iscygpty.c @@ -0,0 +1,185 @@ +/* + * iscygpty.c -- part of ptycheck + * https://github.com/k-takata/ptycheck + * + * Copyright (c) 2015-2017 K.Takata + * + * You can redistribute it and/or modify it under the terms of either + * the MIT license (as described below) or the Vim license. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifdef _WIN32 + +#include +#include +#include +#include + +#ifdef USE_FILEEXTD +/* VC 7.1 or earlier doesn't support SAL. */ +# if !defined(_MSC_VER) || (_MSC_VER < 1400) +# define __out +# define __in +# define __in_opt +# endif +/* Win32 FileID API Library: + * http://www.microsoft.com/en-us/download/details.aspx?id=22599 + * Needed for WinXP. */ +# include +#else /* USE_FILEEXTD */ +/* VC 8 or earlier. */ +# if defined(_MSC_VER) && (_MSC_VER < 1500) +# ifdef ENABLE_STUB_IMPL +# define STUB_IMPL +# else +# error "Win32 FileID API Library is required for VC2005 or earlier." +# endif +# endif +#endif /* USE_FILEEXTD */ + + +#include "iscygpty.h" + +//#define USE_DYNFILEID +#ifdef USE_DYNFILEID +typedef BOOL (WINAPI *pfnGetFileInformationByHandleEx)( + HANDLE hFile, + FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + LPVOID lpFileInformation, + DWORD dwBufferSize +); +static pfnGetFileInformationByHandleEx pGetFileInformationByHandleEx = NULL; + +# ifndef USE_FILEEXTD +static BOOL WINAPI stub_GetFileInformationByHandleEx( + HANDLE hFile, + FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + LPVOID lpFileInformation, + DWORD dwBufferSize + ) +{ + return FALSE; +} +# endif + +static void setup_fileid_api(void) +{ + if (pGetFileInformationByHandleEx != NULL) { + return; + } + pGetFileInformationByHandleEx = (pfnGetFileInformationByHandleEx) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), + "GetFileInformationByHandleEx"); + if (pGetFileInformationByHandleEx == NULL) { +# ifdef USE_FILEEXTD + pGetFileInformationByHandleEx = GetFileInformationByHandleEx; +# else + pGetFileInformationByHandleEx = stub_GetFileInformationByHandleEx; +# endif + } +} +#else +# define pGetFileInformationByHandleEx GetFileInformationByHandleEx +# define setup_fileid_api() +#endif + + +#define is_wprefix(s, prefix) \ + (wcsncmp((s), (prefix), sizeof(prefix) / sizeof(WCHAR) - 1) == 0) + +/* Check if the fd is a cygwin/msys's pty. */ +int is_cygpty(int fd) +{ +#ifdef STUB_IMPL + return 0; +#else + HANDLE h; + int size = sizeof(FILE_NAME_INFO) + sizeof(WCHAR) * (MAX_PATH - 1); + FILE_NAME_INFO *nameinfo; + WCHAR *p = NULL; + + setup_fileid_api(); + + h = (HANDLE) _get_osfhandle(fd); + if (h == INVALID_HANDLE_VALUE) { + return 0; + } + /* Cygwin/msys's pty is a pipe. */ + if (GetFileType(h) != FILE_TYPE_PIPE) { + return 0; + } + nameinfo = malloc(size + sizeof(WCHAR)); + if (nameinfo == NULL) { + return 0; + } + /* Check the name of the pipe: + * '\{cygwin,msys}-XXXXXXXXXXXXXXXX-ptyN-{from,to}-master' */ + if (pGetFileInformationByHandleEx(h, FileNameInfo, nameinfo, size)) { + nameinfo->FileName[nameinfo->FileNameLength / sizeof(WCHAR)] = L'\0'; + p = nameinfo->FileName; + if (is_wprefix(p, L"\\cygwin-")) { /* Cygwin */ + p += 8; + } else if (is_wprefix(p, L"\\msys-")) { /* MSYS and MSYS2 */ + p += 6; + } else { + p = NULL; + } + if (p != NULL) { + while (*p && isxdigit(*p)) /* Skip 16-digit hexadecimal. */ + ++p; + if (is_wprefix(p, L"-pty")) { + p += 4; + } else { + p = NULL; + } + } + if (p != NULL) { + while (*p && isdigit(*p)) /* Skip pty number. */ + ++p; + if (is_wprefix(p, L"-from-master")) { + //p += 12; + } else if (is_wprefix(p, L"-to-master")) { + //p += 10; + } else { + p = NULL; + } + } + } + free(nameinfo); + return (p != NULL); +#endif /* STUB_IMPL */ +} + +/* Check if at least one cygwin/msys pty is used. */ +int is_cygpty_used(void) +{ + int fd, ret = 0; + + for (fd = 0; fd < 3; fd++) { + ret |= is_cygpty(fd); + } + return ret; +} + +#endif /* _WIN32 */ + +/* vim: set ts=4 sw=4: */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f5bea14b16b401..0bbb279511a3e5 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -30,6 +30,7 @@ #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "opcode.h" +#include "iscygpty.h" #include // setlocale() #include // getenv() @@ -3113,7 +3114,7 @@ Py_Exit(int sts) int Py_FdIsInteractive(FILE *fp, const char *filename) { - if (isatty(fileno(fp))) { + if (isatty(fileno(fp)) || is_cygpty(fileno(fp))) { return 1; } if (!_Py_GetConfig()->interactive) { @@ -3128,7 +3129,7 @@ Py_FdIsInteractive(FILE *fp, const char *filename) int _Py_FdIsInteractive(FILE *fp, PyObject *filename) { - if (isatty(fileno(fp))) { + if (isatty(fileno(fp)) || is_cygpty(fileno(fp))) { return 1; } if (!_Py_GetConfig()->interactive) { From f01f0889085c2cf4785c87b9583999966e113b0f Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:16 +0530 Subject: [PATCH 040/127] build: link win resource files and build pythonw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Makefile.pre.in | 26 +++++++++++++++++++++----- configure.ac | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2af2beea09ecc7..d7916604dfc4c2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -38,6 +38,7 @@ CC= @CC@ CXX= @CXX@ LINKCC= @LINKCC@ AR= @AR@ +WINDRES= @WINDRES@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ @@ -124,6 +125,7 @@ PY_CORE_CFLAGS= $(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE PY_CORE_LDFLAGS=$(PY_LDFLAGS) $(PY_LDFLAGS_NODIST) # Strict or non-strict aliasing flags used to compile dtoa.c, see above CFLAGS_ALIASING=@CFLAGS_ALIASING@ +RCFLAGS=@RCFLAGS@ # Machine-dependent subdirectories @@ -286,6 +288,7 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) +BUILDPYTHONW= pythonw$(BUILDEXE) HOSTRUNNER= @HOSTRUNNER@ @@ -622,7 +625,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: all .PHONY: build_all -build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ +build_all: check-clean-src $(BUILDPYTHON) $(BUILDPYTHONW) platform sharedmods \ gdbhooks Programs/_testembed scripts checksharedmods rundsymutil .PHONY: build_wasm @@ -789,9 +792,21 @@ coverage-report: regen-token regen-frozen clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir) +python_exe.o: $(srcdir)/PC/python_exe.rc + $(WINDRES) $(RCFLAGS) -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_exe.rc $@ + +pythonw_exe.o: $(srcdir)/PC/pythonw_exe.rc + $(WINDRES) $(RCFLAGS) -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pythonw_exe.rc $@ + +python_nt.o: $(srcdir)/PC/python_nt.rc + $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ + +$(BUILDPYTHONW): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) pythonw_exe.o + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -mwindows -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) pythonw_exe.o + # Build the interpreter -$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) - $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) +$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) python_exe.o + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) python_exe.o platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -880,10 +895,10 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ # This rule builds the Cygwin Python DLL and import library if configured # for a shared core library; otherwise, this rule is a noop. -$(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) +$(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) python_nt.o if test -n "$(DLLLIBRARY)"; then \ $(LDSHARED) -Wl,--out-implib=$@ -o $(DLLLIBRARY) $^ \ - $(LIBS) $(MODLIBS) $(SYSLIBS); \ + $(LIBS) $(MODLIBS) $(SYSLIBS) python_nt.o; \ else true; \ fi @@ -1979,6 +1994,7 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ done if test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(BINDIR)/python3w$(EXE); \ else \ $(INSTALL_PROGRAM) $(STRIPFLAG) Mac/pythonw $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ fi diff --git a/configure.ac b/configure.ac index 1c8d1e73a7289e..af1f71173bd3bc 100644 --- a/configure.ac +++ b/configure.ac @@ -1762,6 +1762,10 @@ AC_SUBST([LINK_PYTHON_OBJS]) AC_SUBST([AR]) AC_CHECK_TOOLS([AR], [ar aal], [ar]) +# windres program +AC_SUBST(WINDRES) +AC_CHECK_TOOL(WINDRES, windres) + # tweak ARFLAGS only if the user didn't set it on the command line AC_SUBST([ARFLAGS]) if test -z "$ARFLAGS" @@ -7108,10 +7112,33 @@ AC_MSG_RESULT([done]) case $host in *-*-mingw*) LIBS="$LIBS -lversion -lshlwapi -lpathcch" + AC_PROG_AWK + if test "$AWK" = "gawk"; then + awk_extra_flag="--non-decimal-data" + fi + AC_MSG_CHECKING([FIELD3]) + FIELD3=$($AWK $awk_extra_flag '\ + /^#define PY_RELEASE_LEVEL_/ {levels[$2]=$3} \ + /^#define PY_MICRO_VERSION[[:space:]]+/ {micro=$3} \ + /^#define PY_RELEASE_LEVEL[[:space:]]+/ {level=levels[$3]} \ + /^#define PY_RELEASE_SERIAL[[:space:]]+/ {serial=$3} \ + END {print micro * 1000 + level * 10 + serial}' \ + $srcdir/Include/patchlevel.h + ) + + AC_MSG_RESULT([${FIELD3}]) + RCFLAGS="$RCFLAGS -DFIELD3=$FIELD3 -O COFF" + + case $host in + i686*) RCFLAGS="$RCFLAGS --target=pe-i386" ;; + x86_64*) RCFLAGS="$RCFLAGS --target=pe-x86-64" ;; + *) ;; + esac ;; *) ;; esac +AC_SUBST(RCFLAGS) # Availability of -O2: AC_CACHE_CHECK([for -O2], [ac_cv_compile_o2], [ From 32e36bc1162c71b0f9a444645fb2f9c94c9a9c1d Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:18 +0530 Subject: [PATCH 041/127] fix isselectable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Include/fileobject.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/fileobject.h b/Include/fileobject.h index 2deef544d667a5..ae4bdac2edcf97 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -30,7 +30,7 @@ Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_UTF8Mode; #endif /* A routine to check if a file descriptor can be select()-ed. */ -#ifdef _MSC_VER +#ifdef MS_WINDOWS /* On Windows, any socket fd can be select()-ed, no matter how high */ #define _PyIsSelectable_fd(FD) (1) #else From 4f10cdb8782deabae69e1140e184183c55ad146b Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:23 +0530 Subject: [PATCH 042/127] configure.ac: fix inet_pton check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- configure.ac | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure.ac b/configure.ac index af1f71173bd3bc..87e8fc662c938c 100644 --- a/configure.ac +++ b/configure.ac @@ -5403,10 +5403,14 @@ PY_CHECK_NETDB_FUNC([getprotobyname]) dnl PY_CHECK_SOCKET_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_SOCKET_FUNC], [PY_CHECK_FUNC([$1], [ +#ifdef _WIN32 +#include +#else #include #include #include #include +#endif ])]) PY_CHECK_SOCKET_FUNC([inet_aton]) From 0c9d32752bdab2a27c42f2aeca9103d8d1a2fcd5 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:27 +0530 Subject: [PATCH 043/127] pass gen profile ldflags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index d7916604dfc4c2..db921e5a14e1ed 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -898,7 +898,7 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) python_nt.o if test -n "$(DLLLIBRARY)"; then \ $(LDSHARED) -Wl,--out-implib=$@ -o $(DLLLIBRARY) $^ \ - $(LIBS) $(MODLIBS) $(SYSLIBS) python_nt.o; \ + $(LIBS) $(LDFLAGS_NODIST) $(MODLIBS) $(SYSLIBS) python_nt.o; \ else true; \ fi From 08294f9fd95e351c6e7f048c0fcf76665ceb8824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:29 +0530 Subject: [PATCH 044/127] pkg config windows must link ext with python lib --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 87e8fc662c938c..6843218d1e3f28 100644 --- a/configure.ac +++ b/configure.ac @@ -6284,7 +6284,7 @@ AC_MSG_RESULT([$LDVERSION]) # On Android and Cygwin the shared libraries must be linked with libpython. AC_SUBST([LIBPYTHON]) -if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then +if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin" || test "$MACHDEP" = "win32"); then LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" else LIBPYTHON='' From 4d35d4620d6830c93ff75b650acf208bcd731f64 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 17 Jun 2021 18:52:30 +0530 Subject: [PATCH 045/127] importlib bootstrap path sep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Алексей Co-authored-by: Naveen M K --- Lib/importlib/_bootstrap_external.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 9b8a8dfc5aae28..1fbbc552f28d86 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -42,6 +42,10 @@ path_separators = ['\\', '/'] else: path_separators = ['/'] + +if 'MSYSTEM' in _os.environ: + path_separators = path_separators[::-1] + # Assumption made in _path_join() assert all(len(sep) == 1 for sep in path_separators) path_sep = path_separators[0] From e72a2378f85356a22c9be846593c262d68b61b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:33 +0530 Subject: [PATCH 046/127] warnings fixes --- Modules/_winapi.c | 2 +- Modules/posixmodule.c | 2 +- Modules/socketmodule.h | 2 ++ PC/python_exe.rc | 2 +- PC/pythonw_exe.rc | 2 +- PC/winreg.c | 1 + 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index b5d14686d23c97..abb32f0bd8c718 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1127,7 +1127,7 @@ getattributelist(PyObject *obj, const char *name, AttributeList *attribute_list) DWORD err; BOOL result; PyObject *value; - Py_ssize_t handle_list_size; + Py_ssize_t handle_list_size = 0; DWORD attribute_count = 0; SIZE_T attribute_list_size = 0; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 51b3e525403039..b326a142b51e1d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6280,7 +6280,7 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, /*[clinic end generated code: output=cfcac69d027b82cf input=2fbd62a2f228f8f4]*/ { #ifdef MS_WINDOWS - HANDLE hFile; + HANDLE hFile = 0; FILETIME atime, mtime; #else int result; diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index f5ca00450ee92a..1e92327d04f145 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -68,8 +68,10 @@ struct SOCKADDR_BTH_REDEF { */ # ifdef SIO_GET_MULTICAST_FILTER # include /* for SIO_RCVALL */ +#ifndef __MINGW32__ /* resolve by configure */ # define HAVE_ADDRINFO # define HAVE_SOCKADDR_STORAGE +#endif # define HAVE_GETADDRINFO # define HAVE_GETNAMEINFO # define ENABLE_IPV6 diff --git a/PC/python_exe.rc b/PC/python_exe.rc index c3d3bff019895e..dde0e5384201fa 100644 --- a/PC/python_exe.rc +++ b/PC/python_exe.rc @@ -12,7 +12,7 @@ // current versions of Windows. 1 RT_MANIFEST "python.manifest" -1 ICON DISCARDABLE "icons\python.ico" +1 ICON DISCARDABLE "icons/python.ico" ///////////////////////////////////////////////////////////////////////////// diff --git a/PC/pythonw_exe.rc b/PC/pythonw_exe.rc index 38570b74fa3e02..7ce1043298eac4 100644 --- a/PC/pythonw_exe.rc +++ b/PC/pythonw_exe.rc @@ -12,7 +12,7 @@ // current versions of Windows. 1 RT_MANIFEST "python.manifest" -1 ICON DISCARDABLE "icons\pythonw.ico" +1 ICON DISCARDABLE "icons/pythonw.ico" ///////////////////////////////////////////////////////////////////////////// diff --git a/PC/winreg.c b/PC/winreg.c index 368291d9dafa4e..33ec8ffa07dc99 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -831,6 +831,7 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ) case REG_BINARY: /* ALSO handle ALL unknown data types here. Even if we can't support it natively, we should handle the bits. */ + /* fallthrough */ default: if (retDataSize == 0) { obData = Py_NewRef(Py_None); From 114ac5e4c236fd3e0deb3a14071e5369dbc7880a Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sat, 17 Jun 2023 00:21:25 +0530 Subject: [PATCH 047/127] fix build testinternalcapi --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 6843218d1e3f28..6ac579985fac8f 100644 --- a/configure.ac +++ b/configure.ac @@ -7912,7 +7912,7 @@ PY_STDLIB_MOD([_overlapped], [test "$MACHDEP" = "win32"], [], [], dnl test modules PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) -PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes], [], [-DPY3_DLLNAME="\"$DLLLIBRARY\""], []) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) From 852ade1e64a07991efc2d76c87cae3a51a9e3bf1 Mon Sep 17 00:00:00 2001 From: jeremyd2019 <4524874+jeremyd2019@users.noreply.github.com> Date: Thu, 17 Jun 2021 18:52:36 +0530 Subject: [PATCH 048/127] clang arm64 Co-authored-by: Naveen M K --- Python/getcompiler.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 3e31c80bdf4fb1..7e7be712a6fbdd 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -7,7 +7,7 @@ // Note the __clang__ conditional has to come before the __GNUC__ one because // clang pretends to be GCC. -#if defined(__clang__) +#if defined(__clang__) && !defined(_WIN32) #define COMPILER "[Clang " __clang_version__ "]" #elif defined(__GNUC__) /* To not break compatibility with things that determine @@ -18,6 +18,8 @@ #define COMP_SEP " " #if defined(__x86_64__) #define ARCH_SUFFIX " 64 bit (AMD64)" +#elif defined(__aarch64__) +#define ARCH_SUFFIX " 64 bit (ARM64)" #else #define ARCH_SUFFIX " 32 bit" #endif @@ -25,7 +27,14 @@ #define COMP_SEP "\n" #define ARCH_SUFFIX "" #endif +#if defined(__clang__) +#define str(x) #x +#define xstr(x) str(x) +#define COMPILER COMP_SEP "[GCC Clang " xstr(__clang_major__) "." \ + xstr(__clang_minor__) "." xstr(__clang_patchlevel__) ARCH_SUFFIX "]" +#else #define COMPILER COMP_SEP "[GCC " __VERSION__ ARCH_SUFFIX "]" +#endif // Generic fallbacks. #elif defined(__cplusplus) #define COMPILER "[C++]" From f598b5abe85b03030a4586cbf4425e338576f193 Mon Sep 17 00:00:00 2001 From: jeremyd2019 <4524874+jeremyd2019@users.noreply.github.com> Date: Thu, 17 Jun 2021 18:52:37 +0530 Subject: [PATCH 049/127] configure.ac: set MINGW stack reserve --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 6ac579985fac8f..18d5f415ca4e7e 100644 --- a/configure.ac +++ b/configure.ac @@ -3746,6 +3746,11 @@ then VxWorks*) LINKFORSHARED='-Wl,-export-dynamic';; esac + case $host in + *-*-mingw*) + # for https://bugs.python.org/issue40458 on MINGW + LINKFORSHARED="-Wl,--stack,4194304";; + esac fi AC_MSG_RESULT([$LINKFORSHARED]) From a1d212c2c4954021002cc8692a0cbd11ee63baa0 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 18 Jun 2021 12:38:11 +0530 Subject: [PATCH 050/127] Don't use os.pathsep to find EOF not all distributions in win32 have them as \ instead check using sys.platform Signed-off-by: Naveen M K --- Lib/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index fadfccf9f77377..938b0a24959cf2 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -419,7 +419,7 @@ def setquit(): The repr of each object contains a hint at how it works. """ - if os.sep == '\\': + if sys.platform == 'win32': eof = 'Ctrl-Z plus Return' else: eof = 'Ctrl-D (i.e. EOF)' From 3033d58f6f38cba3c977775715b28503f0d221ef Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 23 Jun 2021 18:12:12 +0530 Subject: [PATCH 051/127] Fix extension suffix for c-extensions on mingw Python is compiled with various compilers which previously had same platform tags or extension suffix. This can be error prone while loading c-extensions, so now each compiler or runtime has a different extension suffix. Also, changed all extension to end with .pyd rather than .dll file. Fixes https://github.com/msys2/MINGW-packages/issues/8843 Signed-off-by: Naveen M K --- Makefile.pre.in | 6 ++-- Python/dynload_win.c | 6 ---- configure.ac | 77 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index db921e5a14e1ed..cfe863457083ef 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -174,6 +174,7 @@ CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) # Symbols used for using shared libraries SHLIB_SUFFIX= @SHLIB_SUFFIX@ EXT_SUFFIX= @EXT_SUFFIX@ +PYD_PLATFORM_TAG = @PYD_PLATFORM_TAG@ LDSHARED= @LDSHARED@ $(PY_LDFLAGS) BLDSHARED= @BLDSHARED@ $(PY_CORE_LDFLAGS) LDCXXSHARED= @LDCXXSHARED@ $(PY_LDFLAGS) @@ -1403,8 +1404,7 @@ Python/dynload_hpux.o: $(srcdir)/Python/dynload_hpux.c Makefile Python/dynload_win.o: $(srcdir)/Python/dynload_win.c Makefile $(CC) -c $(PY_CORE_CFLAGS) \ - -DSHLIB_SUFFIX='"$(SHLIB_SUFFIX)"' \ - -DEXT_SUFFIX='"$(EXT_SUFFIX)"' \ + -DPYD_PLATFORM_TAG='"$(PYD_PLATFORM_TAG)"' \ -o $@ $(srcdir)/Python/dynload_win.c Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h @@ -2483,7 +2483,7 @@ libainstall: all scripts @if test "$(STATIC_LIBPYTHON)" = 1; then \ if test -d $(LIBRARY); then :; else \ if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ - if test "$(SHLIB_SUFFIX)" = .dll; then \ + if test "$(SHLIB_SUFFIX)" = .dll -o "$(SHLIB_SUFFIX)" = .pyd; then \ $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBPL) ; \ else \ $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \ diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 4bb90ba198b887..26d6036ba79966 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -29,12 +29,6 @@ #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" const char *_PyImport_DynLoadFiletab[] = { -#ifdef EXT_SUFFIX - EXT_SUFFIX, /* include SOABI flags where is encoded debug */ -#endif -#ifdef SHLIB_SUFFIX - "-abi" PYTHON_ABI_STRING SHLIB_SUFFIX, -#endif PYD_TAGGED_SUFFIX, PYD_UNTAGGED_SUFFIX, NULL diff --git a/configure.ac b/configure.ac index 18d5f415ca4e7e..19b866aae8fda1 100644 --- a/configure.ac +++ b/configure.ac @@ -3488,7 +3488,7 @@ if test -z "$SHLIB_SUFFIX"; then *) SHLIB_SUFFIX=.so;; esac case $host_os in - mingw*) SHLIB_SUFFIX=.dll;; + mingw*) SHLIB_SUFFIX=.pyd;; esac fi AC_MSG_RESULT([$SHLIB_SUFFIX]) @@ -6249,6 +6249,68 @@ esac # check for endianness AC_C_BIGENDIAN +AC_SUBST(PYD_PLATFORM_TAG) +# Special case of PYD_PLATFORM_TAG with python build with mingw. +# Python can with compiled with clang or gcc and linked +# to msvcrt or ucrt. To avoid conflicts between them +# we are selecting the extension as based on the compiler +# and the runtime they link to +# gcc + x86_64 + msvcrt = cp{version number}-x86_64 +# gcc + i686 + msvcrt = cp{version number}-i686 +# gcc + x86_64 + ucrt = cp{version number}-x86_64-ucrt +# clang + x86_64 + ucrt = cp{version number}-x86_64-clang +# clang + i686 + ucrt = cp{version number}-i686-clang + +PYD_PLATFORM_TAG="" +case $host in + *-*-mingw*) + # check if we are linking to ucrt + AC_MSG_CHECKING(whether linking to ucrt) + AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ + #include + #ifndef _UCRT + #error no ucrt + #endif + int main(){ return 0; } + ]])],[linking_to_ucrt=yes],[linking_to_ucrt=no]) + AC_MSG_RESULT($linking_to_ucrt) + ;; +esac +case $host_os in + mingw*) + AC_MSG_CHECKING(PYD_PLATFORM_TAG) + case $host in + i686-*-mingw*) + if test -n "${cc_is_clang}"; then + # it is CLANG32 + PYD_PLATFORM_TAG="mingw_i686_clang" + else + if test $linking_to_ucrt = no; then + PYD_PLATFORM_TAG="mingw_i686" + else + PYD_PLATFORM_TAG="mingw_i686_ucrt" + fi + fi + ;; + x86_64-*-mingw*) + if test -n "${cc_is_clang}"; then + # it is CLANG64 + PYD_PLATFORM_TAG="mingw_x86_64_clang" + else + if test $linking_to_ucrt = no; then + PYD_PLATFORM_TAG="mingw_x86_64" + else + PYD_PLATFORM_TAG="mingw_x86_64_ucrt" + fi + fi + ;; + aarch64-*-mingw*) + PYD_PLATFORM_TAG+="mingw_aarch64" + ;; + esac + AC_MSG_RESULT($PYD_PLATFORM_TAG) +esac + # ABI version string for Python extension modules. This appears between the # periods in shared library file names, e.g. foo..so. It is calculated # from the following attributes which affect the ABI of this Python build (in @@ -6281,7 +6343,12 @@ if test "$Py_DEBUG" = 'true' -a "$with_trace_refs" != "yes"; then fi AC_SUBST([EXT_SUFFIX]) -EXT_SUFFIX=.${SOABI}${SHLIB_SUFFIX} +VERSION_NO_DOTS=$(echo $LDVERSION | tr -d .) +if test -n "${PYD_PLATFORM_TAG}"; then + EXT_SUFFIX=".cp${VERSION_NO_DOTS}-${PYD_PLATFORM_TAG}${SHLIB_SUFFIX}" +else + EXT_SUFFIX=.${SOABI}${SHLIB_SUFFIX} +fi AC_MSG_CHECKING([LDVERSION]) LDVERSION='$(VERSION)$(ABIFLAGS)' @@ -7040,12 +7107,6 @@ case "$ac_cv_computed_gotos" in yes*) AC_DEFINE([HAVE_COMPUTED_GOTOS], [1], [Define if the C compiler supports computed gotos.]) esac -case $host_os in - mingw*) - dnl Synchronized with _PyImport_DynLoadFiletab (dynload_win.c) - dnl Do not use more then one dot on this platform ! - EXT_SUFFIX=-$SOABI$SHLIB_SUFFIX;; -esac case $ac_sys_system in AIX*) From 8593bd4755f7e892e9b4ad5dae1a8bb79632e0c8 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 21 Sep 2021 21:36:58 +0200 Subject: [PATCH 052/127] Change the `get_platform()` method in sysconfig This would possibly fix building wheels when mingw python is used and would be unique to each python same as EXT_SUFFIX. Also, this modifies the `sys.version` argument to include UCRT in it. Signed-off-by: Naveen M K --- Lib/sysconfig.py | 16 ++++++++++++++-- Python/getcompiler.c | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index a0e07b93be6bf6..2a67b846e81b5b 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -813,8 +813,20 @@ def get_platform(): """ if os.name == 'nt': - if 'GCC' in sys.version: - return 'mingw' + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' if 'amd64' in sys.version.lower(): return 'win-amd64' if '(arm)' in sys.version.lower(): diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 7e7be712a6fbdd..275f79997b4241 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -33,8 +33,12 @@ #define COMPILER COMP_SEP "[GCC Clang " xstr(__clang_major__) "." \ xstr(__clang_minor__) "." xstr(__clang_patchlevel__) ARCH_SUFFIX "]" #else +#if defined(_UCRT) +#define COMPILER COMP_SEP "[GCC UCRT " __VERSION__ ARCH_SUFFIX "]" +#else #define COMPILER COMP_SEP "[GCC " __VERSION__ ARCH_SUFFIX "]" #endif +#endif // Generic fallbacks. #elif defined(__cplusplus) #define COMPILER "[C++]" From 3e7467b231de015ec9f31a4982206e9d79f3f948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Thu, 17 Jun 2021 18:52:09 +0530 Subject: [PATCH 053/127] build: Fix ncursesw include lookup Mirror what is already done for libffi; Look it up via pkg-config and use the exported path via sysconfig in setup.py --- Makefile.pre.in | 4 ++++ configure.ac | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index cfe863457083ef..e7c7b89b845597 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -347,6 +347,10 @@ IO_OBJS= \ Modules/_io/bytesio.o \ Modules/_io/stringio.o +########################################################################## + +NCURSESW_INCLUDEDIR= @NCURSESW_INCLUDEDIR@ + ########################################################################## # Parser diff --git a/configure.ac b/configure.ac index 19b866aae8fda1..a566bba8b0b6e9 100644 --- a/configure.ac +++ b/configure.ac @@ -6900,10 +6900,17 @@ AS_VAR_IF([have_panel], [no], [ AC_MSG_RESULT([$have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)]) ]) +if test -n "$PKG_CONFIG"; then + NCURSESW_INCLUDEDIR="`"$PKG_CONFIG" ncursesw --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ *$//'`" +else + NCURSESW_INCLUDEDIR="" +fi +AC_SUBST(NCURSESW_INCLUDEDIR) + # first curses header check ac_save_cppflags="$CPPFLAGS" if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I/usr/include/ncursesw" + CPPFLAGS="$CPPFLAGS -I$NCURSESW_INCLUDEDIR" fi # On Solaris, term.h requires curses.h From 30836e7f007c363a25d2173dd527331a53eab0de Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jul 2021 08:51:32 +0200 Subject: [PATCH 054/127] tests: fix test_bytes %p has different casing with mingw-w64, but it's implementation defined. Change the test to the mingw-w64 variant. --- Lib/test/test_bytes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index b27d43695eef77..f18d38c6dc0e02 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1168,7 +1168,7 @@ def test_from_format(self): if os.name == 'nt': # Windows (MSCRT) - ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) + ptr_format = '0x%0{}x'.format(2 * sizeof_ptr) def ptr_formatter(ptr): return (ptr_format % ptr) else: From fcfa06e4e11f25e039614afeefcead38fbf0b240 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jul 2021 08:52:50 +0200 Subject: [PATCH 055/127] time: fix strftime not raising for invalid year values It's crt specific and not compiler specific. This fixes a test case in test_time --- Modules/timemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 59078263571f7e..4946cdc2f6563a 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -905,7 +905,7 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } -#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) +#if defined(MS_WINDOWS) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); From 5e7f059b5d94cb624764a6b0b42c1186a27e6a80 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jul 2021 18:23:58 +0200 Subject: [PATCH 056/127] ctypes: find_library('c') should return None with ucrt Just like with MSVC. This fixes a test in test_ctypes. --- Lib/ctypes/util.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index c550883e7c7d4b..a5a2a3f5dc2d9d 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -31,6 +31,12 @@ def _get_build_version(): # else we don't know what version of the compiler this is return None + def find_msvcrt_mingw(): + is_ucrt = 'clang' in sys.version.lower() or 'ucrt' in sys.version.lower() + if is_ucrt: + return None + return 'msvcrt.dll' + def find_msvcrt(): """Return the name of the VC runtime dll""" version = _get_build_version() @@ -54,6 +60,9 @@ def find_msvcrt(): def find_library(name): if name in ('c', 'm'): + import sysconfig + if sysconfig.get_platform().startswith('mingw'): + return find_msvcrt_mingw() return find_msvcrt() # See MSDN for the REAL search order. for directory in os.environ['PATH'].split(os.pathsep): From 0a7778d7bcf908304c939705106c3ec8c3699c2e Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 25 Jul 2021 13:54:12 +0200 Subject: [PATCH 057/127] build: Disable checks for dlopen/dlfcn While it is (potentially) available with mingw we don't want to use it, so skip any checks for it. --- configure.ac | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index a566bba8b0b6e9..d3b8dbfec2556a 100644 --- a/configure.ac +++ b/configure.ac @@ -3058,7 +3058,7 @@ AC_DEFINE([STDC_HEADERS], [1], # checks for header files AC_CHECK_HEADERS([ \ - alloca.h asm/types.h bluetooth.h conio.h crypt.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ + alloca.h asm/types.h bluetooth.h conio.h crypt.h direct.h endian.h errno.h fcntl.h grp.h \ ieeefp.h io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ linux/random.h linux/soundcard.h \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pty.h \ @@ -3070,6 +3070,13 @@ AC_CHECK_HEADERS([ \ sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h sys/xattr.h sysexits.h syslog.h \ termios.h util.h utime.h utmp.h \ ]) + +case $host in + *-*-mingw*) ;; + *) AC_CHECK_HEADERS([dlfcn.h]);; +esac + + AC_HEADER_DIRENT AC_HEADER_MAJOR @@ -3815,7 +3822,12 @@ AC_SUBST([PERF_TRAMPOLINE_OBJ]) # checks for libraries AC_CHECK_LIB([sendfile], [sendfile]) -AC_CHECK_LIB([dl], [dlopen]) # Dynamic linking for SunOS/Solaris and SYSV + +case $host in + *-*-mingw*) ;; + *) AC_CHECK_LIB([dl], [dlopen]) ;; # Dynamic linking for SunOS/Solaris and SYSV +esac + AC_CHECK_LIB([dld], [shl_load]) # Dynamic linking for HP-UX @@ -6159,7 +6171,10 @@ AS_VAR_IF([ac_cv_broken_sem_getvalue], [yes], [ ) ]) -AC_CHECK_DECLS([RTLD_LAZY, RTLD_NOW, RTLD_GLOBAL, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_DEEPBIND, RTLD_MEMBER], [], [], [[@%:@include ]]) +case $host in + *-*-mingw*) ;; + *) AC_CHECK_DECLS([RTLD_LAZY, RTLD_NOW, RTLD_GLOBAL, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_DEEPBIND, RTLD_MEMBER], [], [], [[@%:@include ]]) +esac # determine what size digit to use for Python's longs AC_MSG_CHECKING([digit size for Python's longs]) From f661c24c32371a5d30291b4d50f7b3b37893ab7a Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 25 Jul 2021 15:46:01 +0200 Subject: [PATCH 058/127] Fix install location of the import library --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index e7c7b89b845597..fc1ea7d02b0a80 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2488,7 +2488,7 @@ libainstall: all scripts if test -d $(LIBRARY); then :; else \ if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ if test "$(SHLIB_SUFFIX)" = .dll -o "$(SHLIB_SUFFIX)" = .pyd; then \ - $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBPL) ; \ + $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBDIR) ; \ else \ $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \ fi; \ From 4e033763394be7fd3ffbd43c25a64118d0df4634 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 25 Jul 2021 16:53:35 +0200 Subject: [PATCH 059/127] build: Integrate venvlauncher build/installation into the Makefile This is required for venv creation on Windows. Ideally this would use the venv specific launcher (PC/launcher.c), but a copy of main binary seems to work as well for now. --- Makefile.pre.in | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index fc1ea7d02b0a80..073a770591f5e0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -170,6 +170,7 @@ BINLIBDEST= @BINLIBDEST@ LIBDEST= $(SCRIPTDIR)/python$(VERSION) INCLUDEPY= $(INCLUDEDIR)/python$(LDVERSION) CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) +VENVLAUNCHERDIR= $(BINLIBDEST)/venv/scripts/nt # Symbols used for using shared libraries SHLIB_SUFFIX= @SHLIB_SUFFIX@ @@ -290,6 +291,8 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) BUILDPYTHONW= pythonw$(BUILDEXE) +BUILDVENVLAUNCHER= venvlauncher$(BUILDEXE) +BUILDVENVWLAUNCHER= venvwlauncher$(BUILDEXE) HOSTRUNNER= @HOSTRUNNER@ @@ -630,7 +633,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: all .PHONY: build_all -build_all: check-clean-src $(BUILDPYTHON) $(BUILDPYTHONW) platform sharedmods \ +build_all: check-clean-src $(BUILDPYTHON) $(BUILDPYTHONW) $(BUILDVENVLAUNCHER) $(BUILDVENVWLAUNCHER) platform sharedmods \ gdbhooks Programs/_testembed scripts checksharedmods rundsymutil .PHONY: build_wasm @@ -813,6 +816,14 @@ $(BUILDPYTHONW): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) pythonw $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) python_exe.o $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) python_exe.o +# FIXME: build these from PC/launcher.c instead +$(BUILDVENVLAUNCHER): $(BUILDPYTHON) + cp $(BUILDPYTHON) $(BUILDVENVLAUNCHER) + +# FIXME: build these from PC/launcher.c instead +$(BUILDVENVWLAUNCHER): $(BUILDPYTHONW) + cp $(BUILDPYTHONW) $(BUILDVENVWLAUNCHER) + platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -1988,7 +1999,7 @@ sharedinstall: all # This goes into $(exec_prefix) .PHONY: altbininstall altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ - @for i in $(BINDIR) $(LIBDIR); \ + @for i in $(BINDIR) $(LIBDIR) $(VENVLAUNCHERDIR); \ do \ if test ! -d $(DESTDIR)$$i; then \ echo "Creating directory $$i"; \ @@ -1999,6 +2010,8 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ if test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(BINDIR)/python3w$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(VENVLAUNCHERDIR)/python$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(VENVLAUNCHERDIR)/pythonw$(EXE); \ else \ $(INSTALL_PROGRAM) $(STRIPFLAG) Mac/pythonw $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ fi From 9ec9f0c5a7bc9d63913414430187ae56344c954b Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 1 Aug 2021 15:18:50 +0200 Subject: [PATCH 060/127] configure.ac: set _WIN32_WINNT version --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index d3b8dbfec2556a..d6e86040bb7ee7 100644 --- a/configure.ac +++ b/configure.ac @@ -4597,6 +4597,11 @@ done IFS=$as_save_IFS AC_MSG_RESULT([$DBM_CFLAGS $DBM_LIBS]) +case $host in + *-*-mingw*) + CFLAGS_NODIST="$CFLAGS_NODIST -D_WIN32_WINNT=0x0602";; +esac + # Determine if windows modules should be used. AC_SUBST(USE_WIN32_MODULE) USE_WIN32_MODULE='#' From 025f254000599a2531c3ea5beb32e03bc92daeaf Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 1 Aug 2021 18:35:17 +0200 Subject: [PATCH 061/127] configure.ac: don't check for clock_ functions They shouldn't be exposed on Windows and lead to winpthread being linked in --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index d6e86040bb7ee7..e8844317f794e5 100644 --- a/configure.ac +++ b/configure.ac @@ -5540,6 +5540,9 @@ WITH_SAVE_ENV([ ]) ]) +case $host in + *-*-mingw*) ;; + *) AC_CHECK_FUNCS([clock_gettime], [], [ AC_CHECK_LIB([rt], [clock_gettime], [ LIBS="$LIBS -lrt" @@ -5560,6 +5563,8 @@ AC_CHECK_FUNCS([clock_settime], [], [ AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) ]) ]) + ;; +esac AC_CHECK_FUNCS([clock_nanosleep], [], [ AC_CHECK_LIB([rt], [clock_nanosleep], [ From f23f3fc51cd51509dd46f27852b5464972943b7c Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 8 Aug 2021 10:17:35 +0200 Subject: [PATCH 062/127] expanduser: normpath paths coming from env vars This makes sure we get the same paths as with related functions in pathlib. --- Lib/ntpath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index c4855ae11f36f0..f1d33337e41e6f 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -380,7 +380,7 @@ def expanduser(path): if 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] elif not 'HOMEPATH' in os.environ: - return path + return os.path.normpath(path) else: try: drive = os.environ['HOMEDRIVE'] @@ -407,7 +407,7 @@ def expanduser(path): if isinstance(path, bytes): userhome = os.fsencode(userhome) - return userhome + path[i:] + return os.path.normpath(userhome) + path[i:] # Expand paths containing shell variable substitutions. From 1eaade51fb39dde46b814f22ecd27bdfefb5f0d9 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 18 Jun 2021 17:51:59 +0530 Subject: [PATCH 063/127] CI: test the build and add some mingw specific tests - Use actions/setup-python for setting up correct version in cross build - CI: add cross llvm-mingw jobs - CI: fix sed pattern for python-config.py shebang The shebang (`#!`) does not include `${pkgdir}` so this sed did nothing - CI: remove --without-c-locale-coercion No longer needed since the default now works on Windows - Make sure we always use the stdlib distutils - CI: update actions and images ubuntu-18.04 is gone now - CI: updates for mstorsjo/llvm-mingw moving to 20.04 - CI: 3.10 -> 3.11 - CI: remove `--with-system-libmpdec` for cross builds - CI: move gcc cross build to Ubuntu for newer mingw-w64 The build requires headers that are only available in v12 (processsnapshot.h) and Arch is still on v11. So move things to Ubuntu. Similarely to other cross builds remove the system libs configure flags, those were never there and now fail properly with the 3.12 build system. Co-authored-by: Christoph Reiter --- .github/workflows/mingw.yml | 289 +++++++++++++++++++++++++++++++++ mingw_ignorefile.txt | 34 ++++ mingw_smoketests.py | 313 ++++++++++++++++++++++++++++++++++++ 3 files changed, 636 insertions(+) create mode 100644 .github/workflows/mingw.yml create mode 100644 mingw_ignorefile.txt create mode 100644 mingw_smoketests.py diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml new file mode 100644 index 00000000000000..2b606fb992dd0e --- /dev/null +++ b/.github/workflows/mingw.yml @@ -0,0 +1,289 @@ +name: Build +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + msystem: ['MINGW64','MINGW32','UCRT64','CLANG64'] + include: + - msystem: MINGW64 + prefix: mingw-w64-x86_64 + - msystem: MINGW32 + prefix: mingw-w64-i686 + - msystem: UCRT64 + prefix: mingw-w64-ucrt-x86_64 + - msystem: CLANG64 + prefix: mingw-w64-clang-x86_64 + steps: + - name: Setup git + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - uses: actions/checkout@v3 + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + release: false + update: true + install: >- + make + binutils + autoconf + autoconf-archive + automake-wrapper + tar + gzip + ${{ matrix.prefix }}-toolchain + ${{ matrix.prefix }}-expat + ${{ matrix.prefix }}-bzip2 + ${{ matrix.prefix }}-libffi + ${{ matrix.prefix }}-mpdecimal + ${{ matrix.prefix }}-ncurses + ${{ matrix.prefix }}-openssl + ${{ matrix.prefix }}-sqlite3 + ${{ matrix.prefix }}-tcl + ${{ matrix.prefix }}-tk + ${{ matrix.prefix }}-zlib + ${{ matrix.prefix }}-xz + ${{ matrix.prefix }}-tzdata + + - name: Build Python + shell: msys2 {0} + run: | + set -ex + + if [[ "${{ matrix.msystem }}" == CLANG* ]]; then + export CC=clang + export CXX=clang++ + fi + autoreconf -vfi + + rm -Rf _build && mkdir _build && cd _build + + ../configure \ + --prefix=${MINGW_PREFIX} \ + --host=${MINGW_CHOST} \ + --build=${MINGW_CHOST} \ + --enable-shared \ + --with-system-expat \ + --with-system-libmpdec \ + --without-ensurepip \ + --enable-loadable-sqlite-extensions \ + --with-tzpath=${MINGW_PREFIX}/share/zoneinfo \ + --enable-optimizations + + make -j8 + + - name: Run Smoke Test (build) + shell: msys2 {0} + run: | + SMOKETESTS="$(pwd)/mingw_smoketests.py" + cd _build + ./python.exe "$SMOKETESTS" + MSYSTEM= ./python.exe "$SMOKETESTS" + + - name: Run tests + shell: msys2 {0} + run: | + IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + cd _build + MSYSTEM= ./python.exe -m test -j8 --ignorefile "$IGNOREFILE" -W + + - name: Run broken tests + continue-on-error: true + shell: msys2 {0} + run: | + IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + cd _build + MSYSTEM= ./python.exe -m test -j8 --matchfile "$IGNOREFILE" -W + + - name: Install + shell: msys2 {0} + run: | + set -ex + cd _build + + pkgdir=python_pkgdir + + make -j1 install DESTDIR="${pkgdir}" + + # Fix shebangs + _pybasever=$(./python.exe -c "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}');") + for fscripts in 2to3 2to3-${_pybasever} idle3 idle${_pybasever} pydoc3 pydoc${_pybasever}; do + sed -i "s|$(cygpath -w ${MINGW_PREFIX} | sed 's|\\|\\\\|g')/bin/python${_pybasever}.exe|/usr/bin/env python${_pybasever}.exe|g" "${pkgdir}${MINGW_PREFIX}"/bin/${fscripts} + done + sed -i "s|#!${MINGW_PREFIX}/bin/python${_pybasever}.exe|#!/usr/bin/env python${_pybasever}.exe|" "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${_pybasever}/python-config.py + + # Create version-less aliases + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3.exe "${pkgdir}${MINGW_PREFIX}"/bin/python.exe + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3w.exe "${pkgdir}${MINGW_PREFIX}"/bin/pythonw.exe + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3-config "${pkgdir}${MINGW_PREFIX}"/bin/python-config + cp "${pkgdir}${MINGW_PREFIX}"/bin/idle3 "${pkgdir}${MINGW_PREFIX}"/bin/idle + cp "${pkgdir}${MINGW_PREFIX}"/bin/pydoc3 "${pkgdir}${MINGW_PREFIX}"/bin/pydoc + + - name: Run Smoke Test (installed) + shell: msys2 {0} + run: | + export PYTHONTZPATH="${MINGW_PREFIX}/share/zoneinfo" + SMOKETESTS="$(pwd)/mingw_smoketests.py" + cd _build + cd python_pkgdir/${MINGW_PREFIX}/bin + ./python.exe "$SMOKETESTS" + MSYSTEM= ./python.exe "$SMOKETESTS" + + - name: Compress + if: always() + shell: msys2 {0} + run: | + cd _build + tar -zcf python.tar.gz python_pkgdir/ + + - name: Upload + uses: actions/upload-artifact@v3 + if: always() + with: + name: build-${{ matrix.msystem }} + path: _build/python.tar.gz + + cross-gcc-x86_64: + runs-on: ubuntu-latest + container: + image: ubuntu:24.10 + steps: + - uses: actions/checkout@v3 + - name: Install deps + run: | + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + gcc-mingw-w64-x86-64 \ + g++-mingw-w64-x86-64 \ + autoconf-archive \ + autoconf \ + automake \ + zip \ + make \ + pkg-config + + - uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Check Python Version + run: | + which python + python --version + + - name: Build + run: | + autoreconf -vfi + + mkdir _build && cd _build + + ../configure \ + --host=x86_64-w64-mingw32 \ + --build=x86_64-pc-linux-gnu \ + --enable-shared \ + --without-ensurepip \ + --enable-loadable-sqlite-extensions \ + --with-build-python=yes + + make -j8 + + make install DESTDIR="$(pwd)/install" + + - name: 'Zip files' + run: | + zip -r install.zip _build/install + + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: build-cross-gcc-x86_64 + path: install.zip + + cross-gcc-x86_64-test: + needs: [cross-gcc-x86_64] + runs-on: windows-latest + steps: + - uses: actions/download-artifact@v3 + with: + name: build-cross-gcc-x86_64 + + - name: 'Run tests' + run: | + 7z x install.zip + ./_build/install/usr/local/bin/python3.exe -c "import sysconfig, pprint; pprint.pprint(sysconfig.get_config_vars())" + + + cross-llvm-mingw: + runs-on: ubuntu-latest + container: + image: mstorsjo/llvm-mingw:latest + strategy: + fail-fast: false + matrix: + arch: ['x86_64', 'i686', 'aarch64', 'armv7'] + steps: + - uses: actions/checkout@v3 + + - name: Install deps + run: | + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -qqy software-properties-common + add-apt-repository --yes ppa:deadsnakes/ppa + apt-get update -qq + apt-get install -qqy autoconf-archive python3.12-dev python3.12 + + - name: Build + run: | + autoreconf -vfi + + mkdir _build && cd _build + + export CC=${{ matrix.arch }}-w64-mingw32-clang + export CXX=${CC}++ + ../configure \ + --host=${{ matrix.arch }}-w64-mingw32 \ + --build=x86_64-pc-linux-gnu \ + --enable-shared \ + --without-ensurepip \ + --without-c-locale-coercion \ + --enable-loadable-sqlite-extensions \ + --with-build-python=yes + + make -j8 + + make install DESTDIR="$(pwd)/install" + + - name: 'Zip files' + run: | + zip -r install.zip _build/install + + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: build-cross-llvm-mingw-${{ matrix.arch }} + path: install.zip + + cross-llvm-mingw-test: + needs: [cross-llvm-mingw] + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + arch: ['x86_64', 'i686'] + steps: + - uses: actions/download-artifact@v3 + with: + name: build-cross-llvm-mingw-${{ matrix.arch }} + + - name: 'Run tests' + run: | + 7z x install.zip + ./_build/install/usr/local/bin/python3.exe -c "import sysconfig, pprint; pprint.pprint(sysconfig.get_config_vars())" + + diff --git a/mingw_ignorefile.txt b/mingw_ignorefile.txt new file mode 100644 index 00000000000000..dc3802e6c51313 --- /dev/null +++ b/mingw_ignorefile.txt @@ -0,0 +1,34 @@ +ctypes.test.test_loading.LoaderTest.test_load_dll_with_flags +distutils.tests.test_bdist_dumb.BuildDumbTestCase.test_simple_built +distutils.tests.test_cygwinccompiler.CygwinCCompilerTestCase.test_get_versions +distutils.tests.test_util.UtilTestCase.test_change_root +test.datetimetester.TestLocalTimeDisambiguation_Fast.* +test.datetimetester.TestLocalTimeDisambiguation_Pure.* +test.test_cmath.CMathTests.test_specific_values +test.test_cmd_line_script.CmdLineTest.test_consistent_sys_path_for_direct_execution +test.test_compileall.CommandLineTestsNoSourceEpoch.* +test.test_compileall.CommandLineTestsWithSourceEpoch.* +test.test_compileall.CompileallTestsWithoutSourceEpoch.* +test.test_compileall.CompileallTestsWithSourceEpoch.* +test.test_import.ImportTests.test_dll_dependency_import +test.test_math.MathTests.* +test.test_ntpath.NtCommonTest.test_import +test.test_os.StatAttributeTests.test_stat_block_device +test.test_os.TestScandir.test_attributes +test.test_os.UtimeTests.test_large_time +test.test_platform.PlatformTest.test_architecture_via_symlink +test.test_regrtest.ProgramsTestCase.test_pcbuild_rt +test.test_regrtest.ProgramsTestCase.test_tools_buildbot_test +test.test_site._pthFileTests.* +test.test_site.HelperFunctionsTests.* +test.test_site.StartupImportTests.* +test.test_ssl.* +test.test_strptime.CalculationTests.* +test.test_strptime.StrptimeTests.test_weekday +test.test_strptime.TimeRETests.test_compile +test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date +test.test_venv.BasicTest.* +test.test_venv.EnsurePipTest.* +# flaky +test.test__xxsubinterpreters.* +test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe \ No newline at end of file diff --git a/mingw_smoketests.py b/mingw_smoketests.py new file mode 100644 index 00000000000000..d92c74ac248053 --- /dev/null +++ b/mingw_smoketests.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# Copyright 2017 Christoph Reiter +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""The goal of this test suite is collect tests for update regressions +and to test msys2 related modifications like for path handling. +Feel free to extend. +""" + +import os +import unittest +import sysconfig + +if "MSYSTEM" in os.environ: + SEP = "/" +else: + SEP = "\\" + +_UCRT = "clang" in sysconfig.get_platform() or "ucrt" in sysconfig.get_platform() + + +class Tests(unittest.TestCase): + + def test_zoneinfo(self): + # https://github.com/msys2-contrib/cpython-mingw/issues/32 + import zoneinfo + self.assertTrue(any(os.path.exists(p) for p in zoneinfo.TZPATH)) + zoneinfo.ZoneInfo("America/Sao_Paulo") + + def test_userdir_path_sep(self): + # Make sure os.path and pathlib use the same path separators + from unittest import mock + from os.path import expanduser + from pathlib import Path + + profiles = ["C:\\foo", "C:/foo"] + for profile in profiles: + with mock.patch.dict(os.environ, {"USERPROFILE": profile}): + self.assertEqual(expanduser("~"), os.path.normpath(expanduser("~"))) + self.assertEqual(str(Path("~").expanduser()), expanduser("~")) + self.assertEqual(str(Path.home()), expanduser("~")) + + def test_sysconfig_schemes(self): + # https://github.com/msys2/MINGW-packages/issues/9319 + import sysconfig + from distutils.dist import Distribution + from distutils.command.install import install + + names = ['scripts', 'purelib', 'platlib', 'data', 'include'] + for scheme in ["nt", "nt_user"]: + for name in names: + c = install(Distribution({"name": "foobar"})) + c.user = (scheme == "nt_user") + c.finalize_options() + if name == "include": + dist_path = os.path.dirname(getattr(c, "install_" + "headers")) + else: + dist_path = getattr(c, "install_" + name) + sys_path = sysconfig.get_path(name, scheme) + self.assertEqual(dist_path, sys_path, (scheme, name)) + + def test_ctypes_find_library(self): + from ctypes.util import find_library + from ctypes import cdll + self.assertTrue(cdll.msvcrt) + if _UCRT: + self.assertIsNone(find_library('c')) + else: + self.assertEqual(find_library('c'), 'msvcrt.dll') + + def test_ctypes_dlopen(self): + import ctypes + import sys + self.assertEqual(ctypes.RTLD_GLOBAL, 0) + self.assertEqual(ctypes.RTLD_GLOBAL, ctypes.RTLD_LOCAL) + self.assertFalse(hasattr(sys, 'setdlopenflags')) + self.assertFalse(hasattr(sys, 'getdlopenflags')) + self.assertFalse([n for n in dir(os) if n.startswith("RTLD_")]) + + def test_time_no_unix_stuff(self): + import time + self.assertFalse([n for n in dir(time) if n.startswith("clock_")]) + self.assertFalse([n for n in dir(time) if n.startswith("CLOCK_")]) + self.assertFalse([n for n in dir(time) if n.startswith("pthread_")]) + self.assertFalse(hasattr(time, 'tzset')) + + def test_strftime(self): + import time + with self.assertRaises(ValueError): + time.strftime('%Y', (12345,) + (0,) * 8) + + def test_sep(self): + self.assertEqual(os.sep, SEP) + + def test_module_file_path(self): + import asyncio + import zlib + self.assertEqual(zlib.__file__, os.path.normpath(zlib.__file__)) + self.assertEqual(asyncio.__file__, os.path.normpath(asyncio.__file__)) + + def test_importlib_frozen_path_sep(self): + import importlib._bootstrap_external + self.assertEqual(importlib._bootstrap_external.path_sep, SEP) + + def test_os_commonpath(self): + self.assertEqual( + os.path.commonpath( + [os.path.join("C:", os.sep, "foo", "bar"), + os.path.join("C:", os.sep, "foo")]), + os.path.join("C:", os.sep, "foo")) + + def test_pathlib(self): + import pathlib + + p = pathlib.Path("foo") / pathlib.Path("foo") + self.assertEqual(str(p), os.path.normpath(p)) + + def test_modules_import(self): + import sqlite3 + import ssl + import ctypes + import curses + + def test_c_modules_import(self): + import _decimal + + def test_socket_inet_ntop(self): + import socket + self.assertTrue(hasattr(socket, "inet_ntop")) + + def test_socket_inet_pton(self): + import socket + self.assertTrue(hasattr(socket, "inet_pton")) + + def test_multiprocessing_queue(self): + from multiprocessing import Queue + Queue(0) + + #def test_socket_timout_normal_error(self): + # import urllib.request + # from urllib.error import URLError + + # try: + # urllib.request.urlopen( + # 'http://localhost', timeout=0.0001).close() + # except URLError: + # pass + + def test_threads(self): + from concurrent.futures import ThreadPoolExecutor + + with ThreadPoolExecutor(1) as pool: + for res in pool.map(lambda *x: None, range(10000)): + pass + + def test_sysconfig(self): + import sysconfig + # This should be able to execute without exceptions + sysconfig.get_config_vars() + + def test_sqlite_enable_load_extension(self): + # Make sure --enable-loadable-sqlite-extensions is used + import sqlite3 + self.assertTrue(sqlite3.Connection.enable_load_extension) + + def test_venv_creation(self): + import tempfile + import venv + import subprocess + import shutil + with tempfile.TemporaryDirectory() as tmp: + builder = venv.EnvBuilder() + builder.create(tmp) + assert os.path.exists(os.path.join(tmp, "bin", "activate")) + assert os.path.exists(os.path.join(tmp, "bin", "python.exe")) + assert os.path.exists(os.path.join(tmp, "bin", "python3.exe")) + subprocess.check_call([shutil.which("bash.exe"), os.path.join(tmp, "bin", "activate")]) + + def test_has_mktime(self): + from time import mktime, gmtime + mktime(gmtime()) + + def test_platform_things(self): + import sys + import sysconfig + import platform + import importlib.machinery + self.assertEqual(sys.implementation.name, "cpython") + self.assertEqual(sys.platform, "win32") + self.assertTrue(sysconfig.get_platform().startswith("mingw")) + self.assertTrue(sysconfig.get_config_var('SOABI').startswith("cpython-")) + ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') + self.assertTrue(ext_suffix.endswith(".pyd")) + self.assertTrue("mingw" in ext_suffix) + self.assertEqual(sysconfig.get_config_var('SHLIB_SUFFIX'), ".pyd") + ext_suffixes = importlib.machinery.EXTENSION_SUFFIXES + self.assertTrue(ext_suffix in ext_suffixes) + self.assertTrue(".pyd" in ext_suffixes) + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2]))) + self.assertEqual(platform.python_implementation(), "CPython") + self.assertEqual(platform.system(), "Windows") + self.assertTrue(isinstance(sys.api_version, int) and sys.api_version > 0) + + def test_c_ext_build(self): + import tempfile + import sys + import subprocess + import textwrap + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmppro: + subprocess.check_call([sys.executable, "-m", "ensurepip", "--user"]) + with Path(tmppro, "setup.py").open("w") as f: + f.write( + textwrap.dedent( + """\ + from setuptools import setup, Extension + + setup( + name='cwrapper', + version='1.0', + ext_modules=[ + Extension( + 'cwrapper', + sources=['cwrapper.c']), + ], + ) + """ + ) + ) + with Path(tmppro, "cwrapper.c").open("w") as f: + f.write( + textwrap.dedent( + """\ + #include + static PyObject * + helloworld(PyObject *self, PyObject *args) + { + printf("Hello World\\n"); + Py_RETURN_NONE; + } + static PyMethodDef + myMethods[] = { + { "helloworld", helloworld, METH_NOARGS, "Prints Hello World" }, + { NULL, NULL, 0, NULL } + }; + static struct PyModuleDef cwrapper = { + PyModuleDef_HEAD_INIT, + "cwrapper", + "Test Module", + -1, + myMethods + }; + + PyMODINIT_FUNC + PyInit_cwrapper(void) + { + return PyModule_Create(&cwrapper); + } + """ + ) + ) + subprocess.check_call( + [sys.executable, "-c", "import struct"], + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + "wheel", + ], + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + tmppro, + ], + ) + subprocess.check_call( + [sys.executable, "-c", "import cwrapper"], + ) + + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From 04d4ccfa737631413f3c5de013718f909f459c0b Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 12 Oct 2021 18:35:21 +0530 Subject: [PATCH 064/127] Define PY3_DLLNAME to fix build PY3_DLLNAME is used in Python/pathconfig.c but isn't defined. I guess it is useless but it will fix the build. Co-authored-by: jeremyd2019 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e8844317f794e5..2b37fe3f5bd1a8 100644 --- a/configure.ac +++ b/configure.ac @@ -5143,7 +5143,7 @@ then *-*-mingw*) DYNLOADFILE="dynload_win.o" extra_machdep_objs="$extra_machdep_objs PC/dl_nt.o" - CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"'" + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"' -DPY3_DLLNAME='L\"$DLLLIBRARY\"'" ;; esac fi From e6ce651ac8a32677e26f6f019948293b4d89bf85 Mon Sep 17 00:00:00 2001 From: Long Nguyen Date: Sun, 14 Nov 2021 14:55:35 +0700 Subject: [PATCH 065/127] _testconsole.c: Fix casing & path sep --- PC/_testconsole.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PC/_testconsole.c b/PC/_testconsole.c index 3221b985d01ba0..47fbe6feb33832 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -116,7 +116,7 @@ _testconsole_read_output_impl(PyObject *module, PyObject *file) Py_RETURN_NONE; } -#include "clinic\_testconsole.c.h" +#include "clinic/_testconsole.c.h" PyMethodDef testconsole_methods[] = { _TESTCONSOLE_WRITE_INPUT_METHODDEF From d8f8e18cf11fbf9d5374b15644dd388228959e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Fri, 19 Nov 2021 12:18:06 +0200 Subject: [PATCH 066/127] Return consistent architecture markers for python on mingw/armv7 --- Lib/sysconfig.py | 2 ++ Python/getcompiler.c | 2 ++ configure.ac | 3 +++ 3 files changed, 7 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 2a67b846e81b5b..0150c18f452565 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -823,6 +823,8 @@ def get_platform(): return 'mingw_x86_64_clang' if 'arm64' in sys.version.lower(): return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' return 'mingw_i686_clang' if 'amd64' in sys.version.lower(): return 'mingw_x86_64' diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 275f79997b4241..4b0b9b328ba2f8 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -20,6 +20,8 @@ #define ARCH_SUFFIX " 64 bit (AMD64)" #elif defined(__aarch64__) #define ARCH_SUFFIX " 64 bit (ARM64)" +#elif defined(__arm__) +#define ARCH_SUFFIX " 32 bit (ARM)" #else #define ARCH_SUFFIX " 32 bit" #endif diff --git a/configure.ac b/configure.ac index 2b37fe3f5bd1a8..4c757d872edbf0 100644 --- a/configure.ac +++ b/configure.ac @@ -6332,6 +6332,9 @@ case $host_os in aarch64-*-mingw*) PYD_PLATFORM_TAG+="mingw_aarch64" ;; + armv7-*-mingw*) + PYD_PLATFORM_TAG+="mingw_armv7" + ;; esac AC_MSG_RESULT($PYD_PLATFORM_TAG) esac From 01ea756b774c7ae2eacf705a132427e3b84c7803 Mon Sep 17 00:00:00 2001 From: jeremyd2019 Date: Mon, 22 Nov 2021 16:04:41 -0800 Subject: [PATCH 067/127] handle ncursesw pkg-config when cross-compiling strip extra args from pkg-config --cflags-only-I, setup.py is only expecting a single path. Use pkg-config ncursesw include dir even if cross-compiling (PKG_CONFIG_PATH should be set in that case). --- configure.ac | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 4c757d872edbf0..ec3962e1f67d1f 100644 --- a/configure.ac +++ b/configure.ac @@ -6929,7 +6929,7 @@ AS_VAR_IF([have_panel], [no], [ ]) if test -n "$PKG_CONFIG"; then - NCURSESW_INCLUDEDIR="`"$PKG_CONFIG" ncursesw --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ *$//'`" + NCURSESW_INCLUDEDIR="`"$PKG_CONFIG" ncursesw --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ .*$//'`" else NCURSESW_INCLUDEDIR="" fi @@ -6937,9 +6937,7 @@ AC_SUBST(NCURSESW_INCLUDEDIR) # first curses header check ac_save_cppflags="$CPPFLAGS" -if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I$NCURSESW_INCLUDEDIR" -fi +CPPFLAGS="$CPPFLAGS -I$NCURSESW_INCLUDEDIR" # On Solaris, term.h requires curses.h AC_CHECK_HEADERS([term.h], [], [], [ From 99dbe205f23c2e8cd35d0cf5d9e9062490d10282 Mon Sep 17 00:00:00 2001 From: jeremyd2019 Date: Sat, 22 Jan 2022 11:33:00 -0800 Subject: [PATCH 068/127] mingw_smoketests: fix _UCRT condition The prior detection missed ARM (32 and 64) being UCRT. Since there are fewer non-UCRT platforms, list those two instead of trying to extend the list of UCRT. --- mingw_smoketests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index d92c74ac248053..ce95846fa2914f 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -34,7 +34,7 @@ else: SEP = "\\" -_UCRT = "clang" in sysconfig.get_platform() or "ucrt" in sysconfig.get_platform() +_UCRT = sysconfig.get_platform() not in ('mingw_x86_64', 'mingw_i686') class Tests(unittest.TestCase): From 12dd8cedd47e2147df10a7b9bb222743b0d82458 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 19 Jan 2022 19:39:20 +0530 Subject: [PATCH 069/127] Modify `sys.winver` to match upstream With this change `sys.winver` will add the Arch for which python was compiled on, for example in 32-bits, `sys.winver` will be `3.10-32`, for arm32 it would be `3.10-arm32` and so on. See https://github.com/msys2-contrib/cpython-mingw/issues/40 --- configure.ac | 22 ++++++++++++++++++---- mingw_smoketests.py | 9 ++++++++- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index ec3962e1f67d1f..e8d63cde226334 100644 --- a/configure.ac +++ b/configure.ac @@ -5141,10 +5141,24 @@ then esac case $host in *-*-mingw*) - DYNLOADFILE="dynload_win.o" - extra_machdep_objs="$extra_machdep_objs PC/dl_nt.o" - CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"' -DPY3_DLLNAME='L\"$DLLLIBRARY\"'" - ;; + DYNLOADFILE="dynload_win.o" + extra_machdep_objs="$extra_machdep_objs PC/dl_nt.o" + CFLAGS_NODIST="$CFLAGS_NODIST -DPY3_DLLNAME='L\"$DLLLIBRARY\"'" + case $host in + i686*) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"${VERSION}-32\"'" + ;; + armv7*) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"${VERSION}-arm32\"'" + ;; + aarch64*) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"${VERSION}-arm64\"'" + ;; + *) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"'" + ;; + esac + ;; esac fi AC_MSG_RESULT([$DYNLOADFILE]) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index ce95846fa2914f..aa766592062dab 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -214,7 +214,14 @@ def test_platform_things(self): ext_suffixes = importlib.machinery.EXTENSION_SUFFIXES self.assertTrue(ext_suffix in ext_suffixes) self.assertTrue(".pyd" in ext_suffixes) - self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2]))) + if sysconfig.get_platform().startswith('mingw_i686'): + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2])) + '-32') + elif sysconfig.get_platform().startswith('mingw_aarch64'): + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2])) + '-arm64') + elif sysconfig.get_platform().startswith('mingw_armv7'): + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2])) + '-arm32') + else: + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2]))) self.assertEqual(platform.python_implementation(), "CPython") self.assertEqual(platform.system(), "Windows") self.assertTrue(isinstance(sys.api_version, int) and sys.api_version > 0) From 5e148853344f2640abbebb81916f64be369f902c Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 19 Jan 2022 20:01:45 +0530 Subject: [PATCH 070/127] Change user site-packages path to include the environment info This should avoid mixing of user site-packages between python from various environments. Previously, the user site-packages should be located at `~/.local/lib/python3.10` for all environment including 32-bits variants which caused problems with 64-bit trying to load 32-bit extensions. Now this path will be changed to `~/.local/lib/python3.10-`, for example, in CLANG64 this would be `~/.local/lib/python3.10-mingw_x86_64_clang`. Fixes https://github.com/msys2-contrib/cpython-mingw/issues/40 --- Lib/site.py | 28 +++++++++++++++++++++++++--- Lib/sysconfig.py | 26 +++++++++++++++----------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 938b0a24959cf2..c1ecd6ec4739c3 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -310,14 +310,36 @@ def joinuser(*args): return joinuser("~", ".local") +# Copy of sysconfig.get_platform() but only for MinGW +def _get_platform(): + if os.name == 'nt': + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' + return sys.platform # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info - if os.name == 'nt' and not _POSIX_BUILD: - ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\Python{ver_nodot}\\site-packages' + if os.name == 'nt': + if not _POSIX_BUILD: + ver_nodot = sys.winver.replace('.', '') + return f'{userbase}\\Python{ver_nodot}\\site-packages' + return f'{userbase}/lib/python{version[0]}.{version[1]}-{_get_platform()}/site-packages' if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/python/site-packages' diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 0150c18f452565..916c1e82b2d9b7 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -141,20 +141,20 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/lib/python{py_version_short}', - 'platstdlib': '{userbase}/lib/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/python{py_version_short_plat}', + 'platstdlib': '{userbase}/lib/python{py_version_short_plat}', + 'purelib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'include': '{userbase}/include/python{py_version_short_plat}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/python{py_version_short_plat}', + 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short_plat}', + 'purelib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'include': '{userbase}/include/python{py_version_short_plat}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -702,6 +702,10 @@ def _init_config_vars(): _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: _CONFIG_VARS['py_version_nodot_plat'] = '' + if os.name == 'nt' and _POSIX_BUILD: + _CONFIG_VARS['py_version_short_plat'] = f'{_PY_VERSION_SHORT}-{get_platform()}' + else: + _CONFIG_VARS['py_version_short_plat'] = _PY_VERSION_SHORT if os.name == 'nt' and not _POSIX_BUILD: _init_non_posix(_CONFIG_VARS) @@ -789,7 +793,7 @@ def get_config_var(name): """ return get_config_vars().get(name) - +# make sure to change site._get_platform() while changing this function def get_platform(): """Return a string that identifies the current platform. From 077f7447d50d418f35d96e1269e5eed8bc0b51a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Thu, 21 Apr 2022 14:25:51 +0300 Subject: [PATCH 071/127] configure: Include a header in the check for _beginthread Previously, the test tried compiling a call to the _beginthread function without either declaring the function (and its parameters) or including the corresponding header. Since Clang 15 (which still is under development, so this may still change before it's released) [1], implicit function declarations are a hard error by default, when building code in C99 mode (or newer). [1] https://github.com/llvm/llvm-project/commit/7d644e1215b376ec5e915df9ea2eeb56e2d94626 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e8d63cde226334..8a0f79c35c90b4 100644 --- a/configure.ac +++ b/configure.ac @@ -2870,7 +2870,7 @@ AC_MSG_RESULT([$with_nt_threads]) if test $with_nt_threads = yes ; then AC_MSG_CHECKING([whether linking with nt-threads work]) AC_LINK_IFELSE([ - AC_LANG_PROGRAM([[]],[[_beginthread(0, 0, 0);]]) + AC_LANG_PROGRAM([[#include ]],[[_beginthread(0, 0, 0);]]) ], [AC_MSG_RESULT([yes])], [AC_MSG_ERROR([failed to link with nt-threads])]) From d542017523f271326ff0034ad5777c03d6cac07a Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 22 Apr 2022 17:31:19 +0200 Subject: [PATCH 072/127] configure.ac: Default to --without-c-locale-coercion on Windows --with-c-locale-coercion otherwise defaults to yes and enables code that isn't compatible on Windows, mainly because the feature is Unix related. Default to "no" on Windows instead. Fixes #36 --- configure.ac | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 8a0f79c35c90b4..2258bb50c77589 100644 --- a/configure.ac +++ b/configure.ac @@ -5030,11 +5030,14 @@ AC_MSG_RESULT([$with_freelists]) AC_MSG_CHECKING([for --with-c-locale-coercion]) AC_ARG_WITH( [c-locale-coercion], - [AS_HELP_STRING([--with-c-locale-coercion], [enable C locale coercion to a UTF-8 based locale (default is yes)])]) + [AS_HELP_STRING([--with-c-locale-coercion], [enable C locale coercion to a UTF-8 based locale (default is yes on Unix, no on Windows)])]) if test -z "$with_c_locale_coercion" then - with_c_locale_coercion="yes" + case $host in + *-*-mingw*) with_c_locale_coercion="no";; + *) with_c_locale_coercion="yes";; + esac fi if test "$with_c_locale_coercion" != "no" then From 10be5ee7595ba0bf8011b8de375ede06b963d105 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 5 Jun 2022 20:28:53 +0530 Subject: [PATCH 073/127] Fix failing tests - In `test_sysconfig`, ignore `test_sysconfig.TestSysConfig.test_user_similar` test failure. - Copy `get_platform()` from from distutils.utils to test_importlib/test_windows.py. - In `test_tcl`, ignore `test_tcl.TclTest.testLoadWithUNC` test failure. - Disable `test.test_asynchat.TestAsynchat.test_line_terminator2`, seems flaky. - skip some more flaky tests - some basic fixes for test_getpath - test_sysconfig.py: fix tests related to mingw --- Lib/test/test_getpath.py | 3 +++ Lib/test/test_importlib/test_windows.py | 17 +++++++++++++++++ Lib/test/test_sysconfig.py | 8 ++++++-- mingw_ignorefile.txt | 13 +++++++++++-- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 3b613a569b981a..4dc2d305730d67 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -837,6 +837,7 @@ def test_symlink_buildpath_macos(self): ENV_PYTHONHOME="", ENV_PYTHONEXECUTABLE="", ENV___PYVENV_LAUNCHER__="", + ENV_MSYSTEM="", argv0="", py_setpath="", real_executable="", @@ -876,6 +877,7 @@ def __init__(self, *a, argv0=None, config=None, **kw): self.update(DEFAULT_NAMESPACE) self["config"] = DEFAULT_CONFIG.copy() self["os_name"] = "nt" + self["is_mingw"] = 0 self["PLATLIBDIR"] = "DLLs" self["PYWINVER"] = "9.8-XY" self["VPATH"] = r"..\.." @@ -1052,6 +1054,7 @@ def __init__(self, *a, argv0=None, config=None, **kw): self.update(DEFAULT_NAMESPACE) self["config"] = DEFAULT_CONFIG.copy() self["os_name"] = "posix" + self["is_mingw"] = 0 self["PLATLIBDIR"] = "lib" self["WITH_NEXT_FRAMEWORK"] = 0 super().__init__(*a, **kw) diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index 40b8aa1787fe08..03b5a98974f3fa 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -24,6 +24,23 @@ def get_platform(): 'x64' : 'win-amd64', 'arm' : 'win-arm32', } + if os.name == 'nt': + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' if ('VSCMD_ARG_TGT_ARCH' in os.environ and os.environ['VSCMD_ARG_TGT_ARCH'] in TARGET_TO_PLAT): return TARGET_TO_PLAT[os.environ['VSCMD_ARG_TGT_ARCH']] diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 67647e1b787ce7..a883037c1199d2 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -19,7 +19,7 @@ from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes, _main) + _expand_vars, _get_preferred_schemes, _main, _POSIX_BUILD) import _osx_support @@ -197,7 +197,7 @@ def test_nt_venv_scheme(self): self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): - if sys.platform == 'win32': + if not _POSIX_BUILD and sys.platform == 'win32': self.assertEqual( sysconfig.get_path('scripts', scheme='venv'), sysconfig.get_path('scripts', scheme='nt_venv') @@ -388,6 +388,10 @@ def test_user_similar(self): if HAS_USER_BASE: user_path = get_path(name, 'posix_user') expected = os.path.normpath(global_path.replace(base, user, 1)) + if os.name == 'nt' and _POSIX_BUILD: + expected = expected.replace( + f'python{sysconfig.get_python_version()}', + f'python{sysconfig.get_python_version()}-{get_platform()}') # bpo-44860: platlib of posix_user doesn't use sys.platlibdir, # whereas posix_prefix does. if name == 'platlib': diff --git a/mingw_ignorefile.txt b/mingw_ignorefile.txt index dc3802e6c51313..e692d7ff26eb4d 100644 --- a/mingw_ignorefile.txt +++ b/mingw_ignorefile.txt @@ -1,4 +1,4 @@ -ctypes.test.test_loading.LoaderTest.test_load_dll_with_flags +test.test_ctypes.test_loading.LoaderTest.test_load_dll_with_flags distutils.tests.test_bdist_dumb.BuildDumbTestCase.test_simple_built distutils.tests.test_cygwinccompiler.CygwinCCompilerTestCase.test_get_versions distutils.tests.test_util.UtilTestCase.test_change_root @@ -29,6 +29,15 @@ test.test_strptime.TimeRETests.test_compile test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date test.test_venv.BasicTest.* test.test_venv.EnsurePipTest.* +test.test_sysconfig.TestSysConfig.test_user_similar +test.test_tcl.TclTest.testLoadWithUNC +test.test_wmi # flaky test.test__xxsubinterpreters.* -test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe \ No newline at end of file +test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe +test.test_asynchat.TestAsynchat.test_line_terminator2 +test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_gc_aclose_09 +test.test_concurrent_futures.ThreadPoolShutdownTest.test_interpreter_shutdown +test.test_asynchat.TestNotConnected.test_disallow_negative_terminator +test.test_logging.SysLogHandlerTest.* +test.test_logging.IPv6SysLogHandlerTest.* From 7f17ffbeccd24a89830ec34250b8930bda3163d8 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 22 Sep 2022 17:51:33 +0200 Subject: [PATCH 074/127] Don't change os.sep with an empty MSYSTEM env var, not just a missing one Up until now this didn't really happen when calling from cygwin because empty env vars were removed before Python would run. But https://github.com/msys2/msys2-runtime/pull/101 changed that. To avoid breaking users that did something like MSYSTEM= python ... not only check that MSYSTEM isn't set but also that it isn't empty when deciding if os.sep/os.altsep should be switched. Also, guard the msystem env check to execute only on MINGW Co-authored-by: Naveen M K --- Lib/importlib/_bootstrap_external.py | 2 +- Lib/ntpath.py | 2 +- Python/pathconfig.c | 20 +++++++++++--------- mingw_smoketests.py | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 1fbbc552f28d86..a177fa72cfd1c3 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -43,7 +43,7 @@ else: path_separators = ['/'] -if 'MSYSTEM' in _os.environ: +if _os.environ.get('MSYSTEM', ''): path_separators = path_separators[::-1] # Assumption made in _path_join() diff --git a/Lib/ntpath.py b/Lib/ntpath.py index f1d33337e41e6f..1323ac23b2bcba 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -21,7 +21,7 @@ import genericpath from genericpath import * -if sys.platform == "win32" and "MSYSTEM" in os.environ: +if sys.platform == "win32" and os.environ.get("MSYSTEM", ""): sep = '/' altsep = '\\' else: diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 3d684d68ff12df..1195a04fe11437 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -2,7 +2,7 @@ #include "Python.h" #include "marshal.h" // PyMarshal_ReadObjectFromString -#include "osdefs.h" // DELIM +#include "osdefs.h" // DELIM, SEP #include "pycore_initconfig.h" #include "pycore_fileutils.h" #include "pycore_pathconfig.h" @@ -50,7 +50,6 @@ Py_StartsWithW(const wchar_t * str, const wchar_t * prefix) char Py_GetSepA(const char *name) { - char* msystem = (char*)2; /* So that non Windows use / as sep */ static char sep = '\0'; #ifdef _WIN32 /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx @@ -65,12 +64,14 @@ Py_GetSepA(const char *name) if (sep != '\0') return sep; #if defined(__MINGW32__) - msystem = Py_GETENV("MSYSTEM"); -#endif - if (msystem != NULL) + char* msystem = getenv("MSYSTEM"); + if (msystem != NULL && strcmp(msystem, "") != 0) sep = '/'; else sep = '\\'; +#else + sep = SEP; +#endif return sep; } @@ -103,7 +104,6 @@ Py_NormalizeSepsA(char *name) wchar_t Py_GetSepW(const wchar_t *name) { - char* msystem = (char*)2; /* So that non Windows use / as sep */ static wchar_t sep = L'\0'; #ifdef _WIN32 /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx @@ -118,12 +118,14 @@ Py_GetSepW(const wchar_t *name) if (sep != L'\0') return sep; #if defined(__MINGW32__) - msystem = Py_GETENV("MSYSTEM"); -#endif - if (msystem != NULL) + char* msystem = getenv("MSYSTEM"); + if (msystem != NULL && strcmp(msystem, "") != 0) sep = L'/'; else sep = L'\\'; +#else + sep = SEP; +#endif return sep; } diff --git a/mingw_smoketests.py b/mingw_smoketests.py index aa766592062dab..2a8bd160b0042a 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -29,7 +29,7 @@ import unittest import sysconfig -if "MSYSTEM" in os.environ: +if os.environ.get("MSYSTEM", ""): SEP = "/" else: SEP = "\\" From 0aa1973ccba8810cd5a02383f7e94fcd03a46d09 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 13:12:56 +0530 Subject: [PATCH 075/127] def VPATH when compiling `Python/sysmodule.c` --- Makefile.pre.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 073a770591f5e0..a8f358763f8b28 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1425,6 +1425,7 @@ Python/dynload_win.o: $(srcdir)/Python/dynload_win.c Makefile Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h $(CC) -c $(PY_CORE_CFLAGS) \ -DABIFLAGS='"$(ABIFLAGS)"' \ + -DVPATH='"$(VPATH)"' \ $(MULTIARCH_CPPFLAGS) \ -o $@ $(srcdir)/Python/sysmodule.c From 2837bc7be6480ce403564055043cb36c5e620b35 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 13:17:48 +0530 Subject: [PATCH 076/127] Make `_Py_CheckPython3` extern it's declared in headers but defined as static here, remove it also run `_Py_CheckPython3` only when using MSVC --- Python/dynload_win.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 26d6036ba79966..1f90e21a77fecc 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -171,8 +171,7 @@ static char *GetPythonImport (HINSTANCE hModule) Return whether the DLL was found. */ extern HMODULE PyWin_DLLhModule; -static int -_Py_CheckPython3(void) +int _Py_CheckPython3(void) { static int python3_checked = 0; static HANDLE hPython3; @@ -226,9 +225,9 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, dl_funcptr p; char funcname[258], *import_python; -#ifdef Py_ENABLE_SHARED - _Py_CheckPython3(); -#endif /* Py_ENABLE_SHARED */ +#ifdef _MSC_VER + _Py_CheckPython3(); +#endif /* _MSC_VER */ wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL); if (wpathname == NULL) From aa75fc6c4398f67bec02ed7632aca083eaec7457 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 14:20:44 +0530 Subject: [PATCH 077/127] link with bcrypt --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 2258bb50c77589..01344fea13fc78 100644 --- a/configure.ac +++ b/configure.ac @@ -7231,7 +7231,7 @@ AC_MSG_RESULT([done]) # For mingw build need additional library for linking case $host in *-*-mingw*) - LIBS="$LIBS -lversion -lshlwapi -lpathcch" + LIBS="$LIBS -lversion -lshlwapi -lpathcch -lbcrypt" AC_PROG_AWK if test "$AWK" = "gawk"; then awk_extra_flag="--non-decimal-data" From 05873e7473bd34485769d73bacd07e74f42b4812 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 14:29:50 +0530 Subject: [PATCH 078/127] correctly find native python it's required for regen-modules --- configure.ac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 01344fea13fc78..56bd3c1895036d 100644 --- a/configure.ac +++ b/configure.ac @@ -204,9 +204,11 @@ AC_SUBST([FREEZE_MODULE]) AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) +NATIVE_PYTHON_SEARCH_PATH_MINGW=`echo $host | grep -Eq 'mingw*' && echo "$MINGW_PREFIX/bin" || echo $PATH` AC_CHECK_PROGS([PYTHON_FOR_REGEN], [python$PACKAGE_VERSION python3.12 python3.11 python3.10 python3 python], - [python3]) + [python3], + [$NATIVE_PYTHON_SEARCH_PATH_MINGW]) AC_SUBST([PYTHON_FOR_REGEN]) AC_MSG_CHECKING([Python for regen version]) From 5bf2fad675f85bb22fce74cfda951c123659061a Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 18 Jun 2023 14:30:09 +0530 Subject: [PATCH 079/127] Add extra flags for `_bootstrap_python` --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a8f358763f8b28..595792fb4f29a5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1111,7 +1111,7 @@ BOOTSTRAP_HEADERS = \ Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS) _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local - $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \ + $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ -municode -mwindows $(LIBRARY_OBJS_OMIT_FROZEN) \ Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS) From 0732ca86b6fc2de88743f8d80c310f38a8c11347 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 18:32:58 +0530 Subject: [PATCH 080/127] posixmodule: undefine `HAVE_OPENDIR` opendir is detected by configure on mingw-w64, and for some reason things don't work as expected. For example, os.listdir always returns the cwd's directory listing instead of the one specified. By un-defining, this, os.listdir will use the one which uses native windows API. --- Modules/posixmodule.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b326a142b51e1d..53b93d2e424917 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -373,6 +373,12 @@ corresponding Unix manual entries for more information on calls."); #elif defined(__MINGW32__) /* GCC for windows hosts */ /* getlogin is detected by configure on mingw-w64 */ # undef HAVE_GETLOGIN +/* opendir is detected by configure on mingw-w64, and for some reason +things don't work as expected. For example, os.listdir always returns +the cwd's directory listing instead of the one specified. By +un-defining, this, os.listdir will use the one which uses native +windows API. */ +# undef HAVE_OPENDIR /*# define HAVE_GETCWD 1 - detected by configure*/ # define HAVE_GETPPID 1 # define HAVE_GETLOGIN 1 From f05f20c5be464efb0a7277ae68f84f5548a123e7 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 18:43:59 +0530 Subject: [PATCH 081/127] getpath.py: add support for mingw - always normalize the PREFIX to an absolute path - use `/` when MSYSTEM is defined --- Include/pylifecycle.h | 1 + Modules/getpath.c | 9 ++++++++ Modules/getpath.py | 48 +++++++++++++++++++++++++++++++------------ Python/fileutils.c | 41 +++++++++++++++++++++--------------- Python/pathconfig.c | 2 +- 5 files changed, 71 insertions(+), 30 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 50d8b76d74b610..5d292014ff50a7 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -21,6 +21,7 @@ PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); +PyAPI_FUNC(wchar_t) Py_GetAltSepW(const wchar_t *); PyAPI_FUNC(wchar_t) Py_GetSepW(const wchar_t *); PyAPI_FUNC(char) Py_GetSepA(const char *); diff --git a/Modules/getpath.c b/Modules/getpath.c index 0a3100007511b1..de7efb628393e9 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -68,6 +68,7 @@ getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) if (path) { wchar_t *abs; if (_Py_abspath((const wchar_t *)_Py_normpath(path, -1), &abs) == 0 && abs) { + abs = _Py_normpath(abs, -1); r = PyUnicode_FromWideChar(abs, -1); PyMem_RawFree((void *)abs); } else { @@ -881,6 +882,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) #else !decode_to_dict(dict, "os_name", "posix") || #endif +#ifdef __MINGW32__ + !int_to_dict(dict, "is_mingw", 1) || +#else + !int_to_dict(dict, "is_mingw", 0) || +#endif #ifdef WITH_NEXT_FRAMEWORK !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 1) || #else @@ -907,6 +913,9 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !funcs_to_dict(dict, config->pathconfig_warnings) || #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || +#endif +#ifdef __MINGW32__ + !env_to_dict(dict, "ENV_MSYSTEM", 0) || #endif PyDict_SetItemString(dict, "__builtins__", PyEval_GetBuiltins()) < 0 ) { diff --git a/Modules/getpath.py b/Modules/getpath.py index 9913fcba497d30..6867e98eba535f 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -30,6 +30,7 @@ # ** Values known at compile time ** # os_name -- [in] one of 'nt', 'posix', 'darwin' +# is_mingw -- [in] True if targeting MinGW # PREFIX -- [in] sysconfig.get_config_var(...) # EXEC_PREFIX -- [in] sysconfig.get_config_var(...) # PYTHONPATH -- [in] sysconfig.get_config_var(...) @@ -51,6 +52,7 @@ # ENV_PYTHONHOME -- [in] getenv(...) # ENV_PYTHONEXECUTABLE -- [in] getenv(...) # ENV___PYVENV_LAUNCHER__ -- [in] getenv(...) +# ENV_MSYSTEM -- [in] getenv(...) # ** Values calculated at runtime ** # config -- [in/out] dict of the PyConfig structure @@ -185,8 +187,27 @@ ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' DELIM = ':' SEP = '/' + ALTSEP = None -elif os_name == 'nt': +elif os_name == 'nt' and is_mingw: + BUILDDIR_TXT = 'pybuilddir.txt' + BUILD_LANDMARK = 'Modules/Setup.local' + DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] + VENV_LANDMARK = 'pyvenv.cfg' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + DELIM = ';' + if ENV_MSYSTEM: + SEP = '/' + ALTSEP = '\\' + else: + SEP = '\\' + ALTSEP = '/' + +elif os_name == 'nt': # MSVC BUILDDIR_TXT = 'pybuilddir.txt' BUILD_LANDMARK = f'{VPATH}\\Modules\\Setup.local' DEFAULT_PROGRAM_NAME = f'python' @@ -199,6 +220,7 @@ WINREG_KEY = f'SOFTWARE\\Python\\PythonCore\\{PYWINVER}\\PythonPath' DELIM = ';' SEP = '\\' + ALTSEP = '/' # ****************************************************************************** @@ -263,10 +285,10 @@ def search_up(prefix, *landmarks, test=isfile): if not executable: executable = real_executable -if not executable and SEP in program_name: +if not executable and (SEP in program_name or + (ALTSEP and ALTSEP in program_name)): # Resolve partial path program_name against current directory executable = abspath(program_name) - if not executable: # All platforms default to real_executable if known at this # stage. POSIX does not set this value. @@ -497,15 +519,15 @@ def search_up(prefix, *landmarks, test=isfile): except (FileNotFoundError, PermissionError): if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)): build_prefix = joinpath(real_executable_dir, VPATH) - if os_name == 'nt': + if os_name == 'nt' and not is_mingw: # QUIRK: Windows builds need platstdlib_dir to be the executable # dir. Normally the builddir marker handles this, but in this # case we need to correct manually. platstdlib_dir = real_executable_dir if build_prefix: - if os_name == 'nt': - # QUIRK: No searching for more landmarks on Windows + if os_name == 'nt' and not is_mingw: + # QUIRK: No searching for more landmarks on MSVC build_stdlib_prefix = build_prefix else: build_stdlib_prefix = search_up(build_prefix, *BUILDSTDLIB_LANDMARKS) @@ -597,7 +619,7 @@ def search_up(prefix, *landmarks, test=isfile): # Detect exec_prefix by searching from executable for the platstdlib_dir if PLATSTDLIB_LANDMARK and not exec_prefix: - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows always assumed these were the same # gh-100320: Our PYDs are assumed to be relative to the Lib directory # (that is, prefix) rather than the executable (that is, executable_dir) @@ -607,7 +629,7 @@ def search_up(prefix, *landmarks, test=isfile): if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: If DLLs is missing on Windows, don't warn, just assume # that they're in exec_prefix if not platstdlib_dir: @@ -660,7 +682,7 @@ def search_up(prefix, *landmarks, test=isfile): pythonpath.append(abspath(p)) # Then add the default zip file - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows uses the library directory rather than the prefix if library: library_dir = dirname(library) @@ -673,7 +695,7 @@ def search_up(prefix, *landmarks, test=isfile): else: pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) - if os_name == 'nt' and use_environment and winreg: + if (not is_mingw) and os_name == 'nt' and use_environment and winreg: # QUIRK: Windows also lists paths in the registry. Paths are stored # as the default value of each subkey of # {HKCU,HKLM}\Software\Python\PythonCore\{winver}\PythonPath @@ -714,7 +736,7 @@ def search_up(prefix, *landmarks, test=isfile): if not platstdlib_dir and exec_prefix: platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK) - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows generates paths differently if platstdlib_dir: pythonpath.append(platstdlib_dir) @@ -742,8 +764,8 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. -if os_name != 'nt' and build_prefix: - prefix = config.get('prefix') or PREFIX +if (os_name != 'nt' or is_mingw) and build_prefix: + prefix = config.get('prefix') or abspath(PREFIX) exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix diff --git a/Python/fileutils.c b/Python/fileutils.c index 6d3b04ff5bb9e6..50c78f18d779dc 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2189,7 +2189,11 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) } #ifdef MS_WINDOWS - return _PyOS_getfullpathname(path, abspath_p); + if (_PyOS_getfullpathname(path, abspath_p) < 0){ + return -1; + } + *abspath_p = _Py_normpath(*abspath_p, -1); + return 0; #else wchar_t cwd[MAXPATHLEN + 1]; cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; @@ -2401,11 +2405,16 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) wchar_t *minP2 = path; // the beginning of the destination range wchar_t lastC = L'\0'; // the last ljusted character, p2[-1] in most cases + const wchar_t sep = Py_GetSepW(NULL); +#ifdef ALTSEP + const wchar_t altsep = Py_GetAltSepW(NULL); +#endif + #define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) #ifdef ALTSEP -#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP) +#define IS_SEP(x) (*(x) == sep || *(x) == altsep) #else -#define IS_SEP(x) (*(x) == SEP) +#define IS_SEP(x) (*(x) == sep) #endif #define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) @@ -2416,7 +2425,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) path++; } p1 = p2 = minP2 = path; - lastC = SEP; + lastC = sep; } #ifdef MS_WINDOWS // Skip past drive segment and update minP2 @@ -2430,13 +2439,13 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) // and network paths, including the first segment. else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) { int sepCount = 2; - *p2++ = SEP; - *p2++ = SEP; + *p2++ = sep; + *p2++ = sep; p1 += 2; for (; !IS_END(p1) && sepCount; ++p1) { if (IS_SEP(p1)) { --sepCount; - *p2++ = lastC = SEP; + *p2++ = lastC = sep; } else { *p2++ = lastC = *p1; } @@ -2449,7 +2458,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) *p2++ = *p1++; *p2++ = *p1++; minP2 = p2 - 1; // Absolute path has SEP at minP2 - lastC = SEP; + lastC = sep; } #endif /* MS_WINDOWS */ @@ -2457,18 +2466,18 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) for (; !IS_END(p1); ++p1) { wchar_t c = *p1; #ifdef ALTSEP - if (c == ALTSEP) { - c = SEP; + if (c == altsep) { + c = sep; } #endif - if (lastC == SEP) { + if (lastC == sep) { if (c == L'.') { int sep_at_1 = SEP_OR_END(&p1[1]); int sep_at_2 = !sep_at_1 && SEP_OR_END(&p1[2]); if (sep_at_2 && p1[1] == L'.') { wchar_t *p3 = p2; - while (p3 != minP2 && *--p3 == SEP) { } - while (p3 != minP2 && *(p3 - 1) != SEP) { --p3; } + while (p3 != minP2 && *--p3 == sep) { } + while (p3 != minP2 && *(p3 - 1) != sep) { --p3; } if (p2 == minP2 || (p3[0] == L'.' && p3[1] == L'.' && IS_SEP(&p3[2]))) { @@ -2477,7 +2486,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) *p2++ = L'.'; *p2++ = L'.'; lastC = L'.'; - } else if (p3[0] == SEP) { + } else if (p3[0] == sep) { // Absolute path, so absorb segment p2 = p3 + 1; } else { @@ -2488,7 +2497,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) } else { *p2++ = lastC = c; } - } else if (c == SEP) { + } else if (c == sep) { } else { *p2++ = lastC = c; } @@ -2498,7 +2507,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) } *p2 = L'\0'; if (p2 != minP2) { - while (--p2 != minP2 && *p2 == SEP) { + while (--p2 != minP2 && *p2 == sep) { *p2 = L'\0'; } } else { diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 1195a04fe11437..08a4a374ea6023 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -129,7 +129,7 @@ Py_GetSepW(const wchar_t *name) return sep; } -static wchar_t +wchar_t Py_GetAltSepW(const wchar_t *name) { char sep = Py_GetSepW(name); From 4bb41fcf9100a14fce24493cf0d28ce234fe8017 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 20:17:23 +0530 Subject: [PATCH 082/127] Don't build _posixsubprocess on Windows. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 56bd3c1895036d..c1188de2148304 100644 --- a/configure.ac +++ b/configure.ac @@ -7879,7 +7879,6 @@ PY_STDLIB_MOD_SIMPLE([_json]) PY_STDLIB_MOD_SIMPLE([_lsprof]) PY_STDLIB_MOD_SIMPLE([_opcode]) PY_STDLIB_MOD_SIMPLE([_pickle]) -PY_STDLIB_MOD_SIMPLE([_posixsubprocess]) PY_STDLIB_MOD_SIMPLE([_queue]) PY_STDLIB_MOD_SIMPLE([_random]) PY_STDLIB_MOD_SIMPLE([select]) @@ -7932,6 +7931,7 @@ PY_STDLIB_MOD([_scproxy], PY_STDLIB_MOD([spwd], [], [test "$ac_cv_func_getspent" = yes -o "$ac_cv_func_getspnam" = yes]) PY_STDLIB_MOD([syslog], [], [test "$ac_cv_header_syslog_h" = yes]) PY_STDLIB_MOD([termios], [], [test "$ac_cv_header_termios_h" = yes]) +PY_STDLIB_MOD([_posixsubprocess], [], [test "$MACHDEP" != "win32"]) dnl _elementtree loads libexpat via CAPI hook in pyexpat PY_STDLIB_MOD([pyexpat], From a94b38e4741f0db36c1a7fb6254b0187947f2993 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 20:35:27 +0530 Subject: [PATCH 083/127] `_ssl`: link with `ws2_32` --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c1188de2148304..e05eb8f8411813 100644 --- a/configure.ac +++ b/configure.ac @@ -8013,7 +8013,7 @@ PY_STDLIB_MOD([_lzma], [], [test "$have_liblzma" = yes], dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], - [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) + [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS -lws2_32]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) From a0da3ca2051c657b058f913a613c5f55efcabaef Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 23:00:43 +0530 Subject: [PATCH 084/127] Always normalize path in abspath --- Lib/ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 1323ac23b2bcba..15830f903b8050 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -594,7 +594,7 @@ def abspath(path): def abspath(path): """Return the absolute version of a path.""" try: - return _getfullpathname(normpath(path)) + return normpath(_getfullpathname(normpath(path))) except (OSError, ValueError): # See gh-75230, handle outside for cleaner traceback pass From ec73278098a041a371462e3b33b0353a23e9e5ff Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 21 Jun 2023 15:42:39 +0530 Subject: [PATCH 085/127] Include `winsock.h` when checking for netdb function also move `gethostname` to that check --- configure.ac | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index e05eb8f8411813..5a59971522674a 100644 --- a/configure.ac +++ b/configure.ac @@ -5205,7 +5205,7 @@ AC_CHECK_FUNCS([ \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \ - getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \ + getgrnam_r getgrouplist getgroups getitimer getloadavg getlogin \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \ @@ -5432,7 +5432,13 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [have_liblzma=yes], [ ]) dnl PY_CHECK_NETDB_FUNC(FUNCTION) -AC_DEFUN([PY_CHECK_NETDB_FUNC], [PY_CHECK_FUNC([$1], [@%:@include ])]) +AC_DEFUN([PY_CHECK_NETDB_FUNC], [PY_CHECK_FUNC([$1], [ +#ifdef _WIN32 + #include +#else + #include +#endif +])]) PY_CHECK_NETDB_FUNC([hstrerror]) dnl not available in WASI yet @@ -5441,6 +5447,7 @@ PY_CHECK_NETDB_FUNC([getservbyport]) PY_CHECK_NETDB_FUNC([gethostbyname]) PY_CHECK_NETDB_FUNC([gethostbyaddr]) PY_CHECK_NETDB_FUNC([getprotobyname]) +PY_CHECK_NETDB_FUNC([gethostname]) dnl PY_CHECK_SOCKET_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_SOCKET_FUNC], [PY_CHECK_FUNC([$1], [ From 2888fff678ba7e44f9e8bbbb0a0d3bbb9e67c761 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 23 Jun 2023 20:13:23 +0530 Subject: [PATCH 086/127] include `_multiprocessing/semaphore.c` on win32 while building `_multiprocessing` extension also, always build that module on win32 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 5a59971522674a..8462adb859fa0f 100644 --- a/configure.ac +++ b/configure.ac @@ -7897,7 +7897,7 @@ PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules PY_STDLIB_MOD([_multiprocessing], - [], [test "$ac_cv_func_sem_unlink" = "yes"], + [], [test "$ac_cv_func_sem_unlink" = "yes" -o "$MACHDEP" = "win32"], [-I\$(srcdir)/Modules/_multiprocessing]) PY_STDLIB_MOD([_posixshmem], [], [test "$have_posix_shmem" = "yes"], From a778e3ff80ce714a42aa2280b600c844f916e4f8 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 23 Jun 2023 20:15:38 +0530 Subject: [PATCH 087/127] configure: build `mmap` module on win32 --- configure.ac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 8462adb859fa0f..f1947afde4783b 100644 --- a/configure.ac +++ b/configure.ac @@ -7917,7 +7917,9 @@ PY_STDLIB_MOD([fcntl], [], [test "$ac_cv_header_sys_ioctl_h" = "yes" -a "$ac_cv_header_fcntl_h" = "yes"], [], [$FCNTL_LIBS]) PY_STDLIB_MOD([mmap], - [], [test "$ac_cv_header_sys_mman_h" = "yes" -a "$ac_cv_header_sys_stat_h" = "yes"]) + [], m4_flatten([test "$ac_cv_header_sys_mman_h" = "yes" + -a "$ac_cv_header_sys_stat_h" = "yes" + -o "$MACHDEP" = "win32"])) PY_STDLIB_MOD([_socket], [], m4_flatten([test "$ac_cv_header_sys_socket_h" = "yes" -a "$ac_cv_header_sys_types_h" = "yes" From 6c6ab847e6acfc9924ef905fc100404914dcbf6f Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 23 Jun 2023 20:49:15 +0530 Subject: [PATCH 088/127] venv creation fixes --- Lib/venv/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index aeb522c3199ca2..7266f83fef2434 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -13,6 +13,7 @@ import types import shlex +from sysconfig import _POSIX_BUILD CORE_VENV_DEPS = ('pip',) logger = logging.getLogger(__name__) @@ -330,7 +331,7 @@ def setup_python(self, context): if not os.path.islink(path): os.chmod(path, 0o755) else: - if self.symlinks: + if self.symlinks and not _POSIX_BUILD: # For symlinking, we need a complete copy of the root directory # If symlinks fail, you'll get unnecessary copies of files, but # we assume that if you've opted into symlinks on Windows then @@ -354,6 +355,12 @@ def setup_python(self, context): if os.path.lexists(src): copier(src, os.path.join(binpath, suffix)) + if _POSIX_BUILD: + # copy from python/pythonw so the venvlauncher magic in symlink_or_copy triggers + copier(os.path.join(dirname, 'python.exe'), os.path.join(binpath, 'python3.exe')) + copier(os.path.join(dirname, 'python.exe'), os.path.join(binpath, 'python%d.%d.exe' % sys.version_info[:2])) + copier(os.path.join(dirname, 'pythonw.exe'), os.path.join(binpath, 'python3w.exe')) + if sysconfig.is_python_build(): # copy init.tcl for root, dirs, files in os.walk(context.python_dir): @@ -378,6 +385,7 @@ def _call_new_python(self, context, *py_args, **kwargs): env['VIRTUAL_ENV'] = context.env_dir env.pop('PYTHONHOME', None) env.pop('PYTHONPATH', None) + env.pop("MSYSTEM", None) kwargs['cwd'] = context.env_dir kwargs['executable'] = context.env_exec_cmd subprocess.check_output(args, **kwargs) From cd38b416953e377919f2e9395e04779785a90825 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 23 Jun 2023 23:34:01 +0530 Subject: [PATCH 089/127] move the `shutdown` function where `winsock.h` is included --- configure.ac | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f1947afde4783b..305a401dbb8bf2 100644 --- a/configure.ac +++ b/configure.ac @@ -5218,7 +5218,7 @@ AC_CHECK_FUNCS([ \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ setitimer setlocale setpgid setpgrp setpriority setregid setresgid \ - setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ + setresuid setreuid setsid setuid setvbuf sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ @@ -5448,6 +5448,7 @@ PY_CHECK_NETDB_FUNC([gethostbyname]) PY_CHECK_NETDB_FUNC([gethostbyaddr]) PY_CHECK_NETDB_FUNC([getprotobyname]) PY_CHECK_NETDB_FUNC([gethostname]) +PY_CHECK_NETDB_FUNC([shutdown]) dnl PY_CHECK_SOCKET_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_SOCKET_FUNC], [PY_CHECK_FUNC([$1], [ From 672b0b08c964fd164ce94f305d9276f10b8b16fa Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jun 2023 21:47:32 +0200 Subject: [PATCH 090/127] configure.ac: set BUILDEXEEXT and EXEEXT --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 305a401dbb8bf2..b3c3ab020df37e 100644 --- a/configure.ac +++ b/configure.ac @@ -1409,6 +1409,7 @@ AC_ARG_WITH([suffix], [Emscripten/browser*], [EXEEXT=.js], [Emscripten/node*], [EXEEXT=.js], [WASI/*], [EXEEXT=.wasm], + [MINGW*], [EXEEXT=.exe], [EXEEXT=] ) ]) @@ -1435,6 +1436,10 @@ else fi rmdir CaseSensitiveTestDir +AS_CASE([$ac_sys_system], + [MINGW], [BUILDEXEEXT=".exe"] +) + case $ac_sys_system in hp*|HP*) case $CC in From 21653df04700f5026f805373ceef81ec5867004e Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jun 2023 23:42:39 +0200 Subject: [PATCH 091/127] configure.ac: fix building some test modules they are guarded by dlopen being present, but if module loading is enabled is more correct. --- configure.ac | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index b3c3ab020df37e..172c673067f534 100644 --- a/configure.ac +++ b/configure.ac @@ -5174,8 +5174,11 @@ fi AC_MSG_RESULT([$DYNLOADFILE]) if test "$DYNLOADFILE" != "dynload_stub.o" then + have_dynamic_loading=yes AC_DEFINE([HAVE_DYNAMIC_LOADING], [1], [Defined when any dynamic module loading is enabled.]) +else + have_dynamic_loading=no fi # MACHDEP_OBJS can be set to platform-specific object files needed by Python @@ -8047,20 +8050,20 @@ PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes], [], [-DPY3_DLLNAME="\"$DLLLIBRARY\""], []) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) -PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$have_dynamic_loading" = yes]) +PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$have_dynamic_loading" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testconsole], [test "$TEST_MODULES" = yes -a "$MACHDEP" = "win32"]) PY_STDLIB_MOD([_ctypes_test], - [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes], + [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$have_dynamic_loading" = yes], [], [$LIBM]) dnl Limited API template modules. dnl The limited C API is not compatible with the Py_TRACE_REFS macro. dnl Emscripten does not support shared libraries yet. -PY_STDLIB_MOD([xxlimited], [test "$with_trace_refs" = "no"], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no"], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited], [test "$with_trace_refs" = "no"], [test "$have_dynamic_loading" = yes]) +PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no"], [test "$have_dynamic_loading" = yes]) # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) From bd0bf6c544738a7fca9bd8add55e84dfc51d52d4 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 25 Jun 2023 17:20:52 +0530 Subject: [PATCH 092/127] Don't convert `sysconfig.get_config_var('VPATH')` to an absolute path this is required for venv to work properly --- Makefile.pre.in | 1 - configure.ac | 10 +--------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 595792fb4f29a5..1b48cd2d6dfd5f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -146,7 +146,6 @@ datarootdir= @datarootdir@ # Locations needed for semi-native fixup of sysconfig. srcdir_b2h= @srcdir_b2h@ -VPATH_b2h= @VPATH_b2h@ abs_srcdir_b2h= @abs_srcdir_b2h@ abs_builddir_b2h= @abs_builddir_b2h@ prefix_b2h= @prefix_b2h@ diff --git a/configure.ac b/configure.ac index 172c673067f534..9671d724c52e51 100644 --- a/configure.ac +++ b/configure.ac @@ -763,12 +763,8 @@ fi # without setting up the same env and also that the build of Python # itself will fail as the paths are not correct for the host tools. # -# Also, getpath.c uses GetModuleFileNameW (replacing \ with /) and -# compares that with the define VPATH (passed in via command-line) -# to determine whether it's the build- or the installed-Python. -# # To work around these issues a set of _b2h variables are created: -# VPATH_b2h, prefix_b2h, srcdir_b2h, abs_srcdir_b2h +# prefix_b2h, srcdir_b2h, abs_srcdir_b2h # and abs_builddir_b2h # .. where b2h stands for build to host. sysconfig.py replaces path # prefixes matching the non-b2h versions with the b2h equivalents. @@ -795,10 +791,6 @@ AC_DEFUN([ABS_PATH_HOST], AC_SUBST([$1]) ]) -AC_MSG_CHECKING(absolute host location of VPATH) -ABS_PATH_HOST([VPATH_b2h],[srcdir]) -AC_MSG_RESULT([$VPATH_b2h]) - AC_MSG_CHECKING(absolute host location of prefix) ABS_PATH_HOST([prefix_b2h],[prefix]) AC_MSG_RESULT([$prefix_b2h]) From ac3e6cf2a90c822e7e0b8a3f9bc52966ba123133 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 25 Jun 2023 17:24:11 +0530 Subject: [PATCH 093/127] Always convert `/` to `\\` before passing though pathcch functions they don't seems to handle `/` as path separator correctly --- Include/pylifecycle.h | 2 ++ Python/fileutils.c | 22 ++++++++++++++++++---- Python/pathconfig.c | 16 ++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 5d292014ff50a7..21346baf73e2ca 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -28,6 +28,8 @@ PyAPI_FUNC(char) Py_GetSepA(const char *); PyAPI_FUNC(void) Py_NormalizeSepsW(wchar_t *); PyAPI_FUNC(void) Py_NormalizeSepsA(char *); +PyAPI_FUNC(void) Py_NormalizeSepsPathcchW(wchar_t *); + /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level * exit functions. diff --git a/Python/fileutils.c b/Python/fileutils.c index 50c78f18d779dc..aae41433515bf4 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2144,19 +2144,31 @@ int _Py_isabs(const wchar_t *path) { #ifdef MS_WINDOWS + // create a copy of path and replace all forward slashes with backslashes + // pathccskiproot does not handle forward slashes + wchar_t *path_copy = _wcsdup(path); + if (path_copy == NULL) { + return 0; + } + Py_NormalizeSepsPathcchW(path_copy); + const wchar_t *tail; - HRESULT hr = PathCchSkipRoot(path, &tail); - if (FAILED(hr) || path == tail) { + HRESULT hr = PathCchSkipRoot(path_copy, &tail); + if (FAILED(hr) || path_copy == tail) { + free(path_copy); return 0; } - if (tail == &path[1] && (path[0] == SEP || path[0] == ALTSEP)) { + if (tail == &path_copy[1] && (path_copy[0] == SEP || path_copy[0] == ALTSEP)) { // Exclude paths with leading SEP + free(path_copy); return 0; } - if (tail == &path[2] && path[1] == L':') { + if (tail == &path_copy[2] && path_copy[1] == L':') { // Exclude drive-relative paths (e.g. C:filename.ext) + free(path_copy); return 0; } + free(path_copy); return 1; #else return (path[0] == SEP); @@ -2303,6 +2315,8 @@ join_relfile(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, const wchar_t *relfile) { #ifdef MS_WINDOWS + Py_NormalizeSepsPathcchW(dirname); + Py_NormalizeSepsPathcchW(relfile); if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, PATHCCH_ALLOW_LONG_PATHS))) { return -1; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 08a4a374ea6023..8053b91d8c559a 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -155,6 +155,22 @@ Py_NormalizeSepsW(wchar_t *name) } } +void +Py_NormalizeSepsPathcchW(wchar_t *name) +{ +#ifdef MS_WINDOWS + assert(name != NULL); + wchar_t sep = '\\'; + wchar_t altsep = '/'; + wchar_t* seps; + seps = wcschr(name, altsep); + while(seps) { + *seps = sep; + seps = wcschr(seps, altsep); + } +#endif +} + /* External interface */ /* Stored values set by C API functions */ From cda1f3666796290ea37cc84806ac03ebde4a945d Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 25 Jun 2023 17:26:55 +0530 Subject: [PATCH 094/127] Build venvlauncher.exe from PC/launcher.c - Use function available in original msvcrt.dll _wdupenv_s -> _wgetenv_s fread_s -> fread - Add a test for checking the new launchers --- Makefile.pre.in | 20 ++++++++++++-------- PC/launcher.c | 29 +++++++++++++++++------------ PC/pylauncher.rc | 20 ++++++++++---------- mingw_smoketests.py | 12 ++++++++++++ 4 files changed, 51 insertions(+), 30 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 1b48cd2d6dfd5f..2d2a64fc6ce693 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -808,6 +808,12 @@ pythonw_exe.o: $(srcdir)/PC/pythonw_exe.rc python_nt.o: $(srcdir)/PC/python_nt.rc $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ +venvlauncher.o: $(srcdir)/PC/pylauncher.rc + $(WINDRES) $(RCFLAGS) -DPY_ICON -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pylauncher.rc $@ + +venvwlauncher.o: $(srcdir)/PC/pylauncher.rc + $(WINDRES) $(RCFLAGS) -DPYW_ICON -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pylauncher.rc $@ + $(BUILDPYTHONW): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) pythonw_exe.o $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -mwindows -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) pythonw_exe.o @@ -815,13 +821,11 @@ $(BUILDPYTHONW): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) pythonw $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) python_exe.o $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) python_exe.o -# FIXME: build these from PC/launcher.c instead -$(BUILDVENVLAUNCHER): $(BUILDPYTHON) - cp $(BUILDPYTHON) $(BUILDVENVLAUNCHER) +$(BUILDVENVLAUNCHER): $(BUILDPYTHON) venvlauncher.o $(srcdir)/PC/launcher.c + $(LINKCC) -D_CONSOLE -DVENV_REDIRECT $(PY_STDMODULE_CFLAGS) -municode -static -static-libgcc -static-libstdc++ venvlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion -# FIXME: build these from PC/launcher.c instead -$(BUILDVENVWLAUNCHER): $(BUILDPYTHONW) - cp $(BUILDPYTHONW) $(BUILDVENVWLAUNCHER) +$(BUILDVENVWLAUNCHER): $(BUILDPYTHONW) venvwlauncher.o $(srcdir)/PC/launcher.c + $(LINKCC) -D_WINDOWS -DVENV_REDIRECT $(PY_STDMODULE_CFLAGS) -mwindows -municode -static -static-libgcc -static-libstdc++ venvwlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -2010,8 +2014,8 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ if test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(BINDIR)/python3w$(EXE); \ - $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(VENVLAUNCHERDIR)/python$(EXE); \ - $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(VENVLAUNCHERDIR)/pythonw$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDVENVLAUNCHER) $(DESTDIR)$(VENVLAUNCHERDIR)/python$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDVENVWLAUNCHER) $(DESTDIR)$(VENVLAUNCHERDIR)/pythonw$(EXE); \ else \ $(INSTALL_PROGRAM) $(STRIPFLAG) Mac/pythonw $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ fi diff --git a/PC/launcher.c b/PC/launcher.c index dc265533740b67..14d8b8fff83d0c 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -924,7 +924,7 @@ static COMMAND path_command; static COMMAND * find_on_path(wchar_t * name) { wchar_t * pathext; - size_t varsize; + size_t requiredSize; wchar_t * context = NULL; wchar_t * extension; COMMAND * result = NULL; @@ -941,18 +941,23 @@ static COMMAND * find_on_path(wchar_t * name) } else { /* No extension - search using registered extensions. */ - rc = _wdupenv_s(&pathext, &varsize, L"PATHEXT"); - if (rc == 0) { - extension = wcstok_s(pathext, L";", &context); - while (extension) { - len = SearchPathW(NULL, name, extension, MSGSIZE, path_command.value, NULL); - if (len) { - result = &path_command; - break; + _wgetenv_s(&requiredSize, NULL, 0, L"PATHEXT"); + if (requiredSize > 0) { + pathext = (wchar_t *)malloc(requiredSize * sizeof(wchar_t)); + /* No extension - search using registered extensions. */ + rc = _wgetenv_s(&requiredSize, pathext, requiredSize, L"PATHEXT"); + if (rc == 0) { + extension = wcstok_s(pathext, L";", &context); + while (extension) { + len = SearchPathW(NULL, name, extension, MSGSIZE, path_command.value, NULL); + if (len) { + result = &path_command; + break; + } + extension = wcstok_s(NULL, L";", &context); } - extension = wcstok_s(NULL, L";", &context); + free(pathext); } - free(pathext); } } return result; @@ -1912,7 +1917,7 @@ process(int argc, wchar_t ** argv) if (_wfopen_s(&f, venv_cfg_path, L"r")) { error(RC_BAD_VENV_CFG, L"Cannot read '%ls'", venv_cfg_path); } - cb = fread_s(buffer, sizeof(buffer), sizeof(buffer[0]), + cb = fread(buffer, sizeof(buffer[0]), sizeof(buffer) / sizeof(buffer[0]), f); fclose(f); diff --git a/PC/pylauncher.rc b/PC/pylauncher.rc index 11862643aa6989..84b291722f6633 100644 --- a/PC/pylauncher.rc +++ b/PC/pylauncher.rc @@ -12,17 +12,17 @@ 1 RT_MANIFEST "python.manifest" #if defined(PY_ICON) -1 ICON DISCARDABLE "icons\python.ico" +1 ICON DISCARDABLE "icons/python.ico" #elif defined(PYW_ICON) -1 ICON DISCARDABLE "icons\pythonw.ico" +1 ICON DISCARDABLE "icons/pythonw.ico" #else -1 ICON DISCARDABLE "icons\launcher.ico" -2 ICON DISCARDABLE "icons\py.ico" -3 ICON DISCARDABLE "icons\pyc.ico" -4 ICON DISCARDABLE "icons\pyd.ico" -5 ICON DISCARDABLE "icons\python.ico" -6 ICON DISCARDABLE "icons\pythonw.ico" -7 ICON DISCARDABLE "icons\setup.ico" +1 ICON DISCARDABLE "icons/launcher.ico" +2 ICON DISCARDABLE "icons/py.ico" +3 ICON DISCARDABLE "icons/pyc.ico" +4 ICON DISCARDABLE "icons/pyd.ico" +5 ICON DISCARDABLE "icons/python.ico" +6 ICON DISCARDABLE "icons/pythonw.ico" +7 ICON DISCARDABLE "icons/setup.ico" #endif 1 USAGE "launcher-usage.txt" @@ -64,4 +64,4 @@ BEGIN BEGIN VALUE "Translation", 0x0, 1200 END -END \ No newline at end of file +END diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 2a8bd160b0042a..00601f956d9267 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -194,6 +194,18 @@ def test_venv_creation(self): assert os.path.exists(os.path.join(tmp, "bin", "python3.exe")) subprocess.check_call([shutil.which("bash.exe"), os.path.join(tmp, "bin", "activate")]) + # This will not work in in-tree build + if not sysconfig.is_python_build(): + op = subprocess.check_output( + [ + os.path.join(tmp, "bin", "python.exe"), + "-c", + "print('Hello World')" + ], + cwd=tmp, + ) + assert op.decode().strip() == "Hello World" + def test_has_mktime(self): from time import mktime, gmtime mktime(gmtime()) From 16a48fca4ea52fcef0e8eda8ab3c0e27428e8176 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 25 Jun 2023 19:39:30 +0530 Subject: [PATCH 095/127] getpath.py: fix `dirname` also, fix finding prefix when in a venv --- Modules/getpath.c | 12 ++++++++++++ Modules/getpath.py | 3 +++ 2 files changed, 15 insertions(+) diff --git a/Modules/getpath.c b/Modules/getpath.c index de7efb628393e9..099cffc62568c3 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -89,6 +89,12 @@ getpath_basename(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); +#ifdef ALTSEP + if (pos < 0) { + // try using altsep + pos = PyUnicode_FindChar(path, ALTSEP, 0, end, -1); + } +#endif if (pos < 0) { return Py_NewRef(path); } @@ -105,6 +111,12 @@ getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); +#ifdef ALTSEP + if (pos < 0) { + // try using altsep + pos = PyUnicode_FindChar(path, ALTSEP, 0, end, -1); + } +#endif if (pos < 0) { return PyUnicode_FromStringAndSize(NULL, 0); } diff --git a/Modules/getpath.py b/Modules/getpath.py index 6867e98eba535f..5687e4f38de37f 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -574,6 +574,9 @@ def search_up(prefix, *landmarks, test=isfile): # First try to detect prefix by looking alongside our runtime library, if known if library and not prefix: library_dir = dirname(library) + if os_name == 'nt' and is_mingw: + # QUIRK: On Windows, mingw Python DLLs are in the bin directory + library_dir = joinpath(library_dir, '..') if ZIP_LANDMARK: if os_name == 'nt': # QUIRK: Windows does not search up for ZIP file From f8523063e6d99d4d8a623825e86e3d62e47a9271 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 28 Jul 2023 20:13:58 +0200 Subject: [PATCH 096/127] getpath: use normpath on all generated paths Instead of just calling normpath in abspath just call normpath on all the config results. This makes sure we don't change getpath.py too much and still cover all outputs. This fixes sys.exec_prefix not matching sys.prefix, see https://github.com/msys2-contrib/cpython-mingw/issues/142 --- Lib/test/test_getpath.py | 6 ++++++ Modules/getpath.c | 21 ++++++++++++++++++++- Modules/getpath.py | 30 ++++++++++++++++-------------- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 4dc2d305730d67..f63e4204a78a2e 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -913,6 +913,9 @@ def __missing__(self, key): except AttributeError: raise KeyError(key) from None + def normpath(self, path): + return ntpath.normpath(path) + def abspath(self, path): if self.isabs(path): return path @@ -1091,6 +1094,9 @@ def __missing__(self, key): except AttributeError: raise KeyError(key) from None + def normpath(self, path): + return path + def abspath(self, path): if self.isabs(path): return path diff --git a/Modules/getpath.c b/Modules/getpath.c index 099cffc62568c3..47b89ebcf522ed 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -54,6 +54,25 @@ /* HELPER FUNCTIONS for getpath.py */ +static PyObject * +getpath_normpath(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } + Py_ssize_t len; + wchar_t *buffer = PyUnicode_AsWideCharString(pathobj, &len); + if (!buffer) { + return NULL; + } + r = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + PyMem_Free(buffer); + return r; +} + static PyObject * getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) { @@ -68,7 +87,6 @@ getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) if (path) { wchar_t *abs; if (_Py_abspath((const wchar_t *)_Py_normpath(path, -1), &abs) == 0 && abs) { - abs = _Py_normpath(abs, -1); r = PyUnicode_FromWideChar(abs, -1); PyMem_RawFree((void *)abs); } else { @@ -523,6 +541,7 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) static PyMethodDef getpath_methods[] = { + {"normpath", getpath_normpath, METH_VARARGS, NULL}, {"abspath", getpath_abspath, METH_VARARGS, NULL}, {"basename", getpath_basename, METH_VARARGS, NULL}, {"dirname", getpath_dirname, METH_VARARGS, NULL}, diff --git a/Modules/getpath.py b/Modules/getpath.py index 5687e4f38de37f..d143f9776eff31 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -233,6 +233,8 @@ def search_up(prefix, *landmarks, test=isfile): return prefix prefix = dirname(prefix) +def _normpath(p): + return normpath(p) if p is not None else None # ****************************************************************************** # READ VARIABLES FROM config @@ -670,7 +672,7 @@ def search_up(prefix, *landmarks, test=isfile): if py_setpath: # If Py_SetPath was called then it overrides any existing search path - config['module_search_paths'] = py_setpath.split(DELIM) + config['module_search_paths'] = [_normpath(p) for p in py_setpath.split(DELIM)] config['module_search_paths_set'] = 1 elif not pythonpath_was_set: @@ -757,7 +759,7 @@ def search_up(prefix, *landmarks, test=isfile): if platstdlib_dir: pythonpath.append(platstdlib_dir) - config['module_search_paths'] = pythonpath + config['module_search_paths'] = [_normpath(p) for p in pythonpath] config['module_search_paths_set'] = 1 @@ -792,23 +794,23 @@ def search_up(prefix, *landmarks, test=isfile): warn("unsupported 'import' line in ._pth file") else: pythonpath.append(joinpath(pth_dir, line)) - config['module_search_paths'] = pythonpath + config['module_search_paths'] = [_normpath(p) for p in pythonpath] config['module_search_paths_set'] = 1 # ****************************************************************************** # UPDATE config FROM CALCULATED VALUES # ****************************************************************************** -config['program_name'] = program_name -config['home'] = home -config['executable'] = executable -config['base_executable'] = base_executable -config['prefix'] = prefix -config['exec_prefix'] = exec_prefix -config['base_prefix'] = base_prefix or prefix -config['base_exec_prefix'] = base_exec_prefix or exec_prefix +config['program_name'] = _normpath(program_name) +config['home'] = _normpath(home) +config['executable'] = _normpath(executable) +config['base_executable'] = _normpath(base_executable) +config['prefix'] = _normpath(prefix) +config['exec_prefix'] = _normpath(exec_prefix) +config['base_prefix'] = _normpath(base_prefix or prefix) +config['base_exec_prefix'] = _normpath(base_exec_prefix or exec_prefix) -config['platlibdir'] = platlibdir +config['platlibdir'] = _normpath(platlibdir) # test_embed expects empty strings, not None -config['stdlib_dir'] = stdlib_dir or '' -config['platstdlib_dir'] = platstdlib_dir or '' +config['stdlib_dir'] = _normpath(stdlib_dir or '') +config['platstdlib_dir'] = _normpath(platstdlib_dir or '') From 00d11588b95d175ecdaa641d3c90c955912c42dd Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 28 Jul 2023 20:14:38 +0200 Subject: [PATCH 097/127] pathconfig: normpath sys.path[0] this gets added as first element in sys.path. use normpath to make sure it uses the right path separator --- Python/pathconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 8053b91d8c559a..7eb9006be5b40a 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -662,7 +662,7 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p) } #endif /* All others */ - PyObject *path0_obj = PyUnicode_FromWideChar(path0, n); + PyObject *path0_obj = PyUnicode_FromWideChar(_Py_normpath(path0, -1), n); if (path0_obj == NULL) { return -1; } From 99d67c56451bfad7961e7a06d8fc16a21d3f65e6 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 28 Jul 2023 20:15:01 +0200 Subject: [PATCH 098/127] smoketests: add some tests for sys/site paths --- mingw_smoketests.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 00601f956d9267..516005c9b3eeb6 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -238,6 +238,29 @@ def test_platform_things(self): self.assertEqual(platform.system(), "Windows") self.assertTrue(isinstance(sys.api_version, int) and sys.api_version > 0) + def test_sys_getpath(self): + # everything sourced from getpath.py + import sys + + def assertNormpath(path): + self.assertEqual(path, os.path.normpath(path)) + + assertNormpath(sys.executable) + assertNormpath(sys._base_executable) + assertNormpath(sys.prefix) + assertNormpath(sys.base_prefix) + assertNormpath(sys.exec_prefix) + assertNormpath(sys.base_exec_prefix) + assertNormpath(sys.platlibdir) + assertNormpath(sys._stdlib_dir) + for p in sys.path: + assertNormpath(p) + + def test_site(self): + import site + + self.assertEqual(len(site.getsitepackages()), 1) + def test_c_ext_build(self): import tempfile import sys From 480b3a1ff99558ce4bec2c61a792a47ee43637e7 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sat, 5 Aug 2023 14:10:27 +0530 Subject: [PATCH 099/127] Search DLLs only on paths added using `add_dll_directory()`. This is the default behavior on upstream Python. We previously reverted this behavior because it broke some use cases. The old behavior can be restored by setting the environment variable PYTHONLEGACYWINDOWSDLLLOADING to 1. Fixes https://github.com/msys2-contrib/cpython-mingw/issues/48 Also fixes https://github.com/msys2-contrib/cpython-mingw/issues/141 Signed-off-by: Naveen M K --- .github/workflows/mingw.yml | 10 +++++----- Lib/test/__main__.py | 8 ++++++++ Python/dynload_win.c | 16 ++++++++++++++-- Tools/build/check_extension_modules.py | 4 ++++ mingw_smoketests.py | 3 +++ 5 files changed, 34 insertions(+), 7 deletions(-) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 2b606fb992dd0e..0c9f644d84cca5 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -124,15 +124,15 @@ jobs: cp "${pkgdir}${MINGW_PREFIX}"/bin/idle3 "${pkgdir}${MINGW_PREFIX}"/bin/idle cp "${pkgdir}${MINGW_PREFIX}"/bin/pydoc3 "${pkgdir}${MINGW_PREFIX}"/bin/pydoc + # copy to / + cp -rf "${pkgdir}"/* / + - name: Run Smoke Test (installed) shell: msys2 {0} run: | - export PYTHONTZPATH="${MINGW_PREFIX}/share/zoneinfo" SMOKETESTS="$(pwd)/mingw_smoketests.py" - cd _build - cd python_pkgdir/${MINGW_PREFIX}/bin - ./python.exe "$SMOKETESTS" - MSYSTEM= ./python.exe "$SMOKETESTS" + ${MINGW_PREFIX}/bin/python.exe "$SMOKETESTS" + MSYSTEM= ${MINGW_PREFIX}/bin/python.exe "$SMOKETESTS" - name: Compress if: always() diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index 82b50ad2c6e777..25bac39e24af20 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,2 +1,10 @@ +import os +import sys + from test.libregrtest.main import main + +if sys.platform == "win32": + # Enable DLL loading from PATH. + os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" + main(_add_python_opts=True) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 1f90e21a77fecc..db7968742b2228 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_add_relfile() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_initconfig.h" #ifdef HAVE_DIRECT_H #include @@ -225,6 +226,18 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, dl_funcptr p; char funcname[258], *import_python; + int use_legacy = 0; + DWORD load_library_flags = 0; + + _Py_get_env_flag(1, &use_legacy, "PYTHONLEGACYWINDOWSDLLLOADING"); + + if (use_legacy) { + load_library_flags = LOAD_WITH_ALTERED_SEARCH_PATH; + } else { + load_library_flags = LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR; + } + #ifdef _MSC_VER _Py_CheckPython3(); #endif /* _MSC_VER */ @@ -249,8 +262,7 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, AddDllDirectory function. We add SEARCH_DLL_LOAD_DIR to ensure DLLs adjacent to the PYD are preferred. */ Py_BEGIN_ALLOW_THREADS - hDLL = LoadLibraryExW(wpathname, NULL, - LOAD_WITH_ALTERED_SEARCH_PATH); + hDLL = LoadLibraryExW(wpathname, NULL, load_library_flags); Py_END_ALLOW_THREADS PyMem_Free(wpathname); diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index aa1ade71a8e937..7ded0f464c205d 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -61,6 +61,10 @@ "winsound", } +if sys.platform == "win32": + # Enable DLL loading from PATH. + os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" + logger = logging.getLogger(__name__) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 516005c9b3eeb6..ca1f652caeec27 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -34,6 +34,9 @@ else: SEP = "\\" +if sysconfig.is_python_build(): + os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" + _UCRT = sysconfig.get_platform() not in ('mingw_x86_64', 'mingw_i686') From 016327ca53842e76a7105f02220b987411f44d6d Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Mon, 21 Aug 2023 08:19:28 +0200 Subject: [PATCH 100/127] Build and install libpython3.dll This provides the limited ABI as a separate DLL, forwarding to the real one. This makes linking with "-lpython3" work. Fixes #147 --- Makefile.pre.in | 15 +++++++++++++-- PC/python3dll.c | 8 ++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2d2a64fc6ce693..e9ded19a3fb519 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -273,6 +273,8 @@ LIBRARY_DEPS= @LIBRARY_DEPS@ LINK_PYTHON_DEPS=@LINK_PYTHON_DEPS@ PY_ENABLE_SHARED= @PY_ENABLE_SHARED@ STATIC_LIBPYTHON= @STATIC_LIBPYTHON@ +ABI3DLLLIBRARY= libpython3.dll +ABI3LDLIBRARY= libpython3.dll.a LIBS= @LIBS@ @@ -633,7 +635,7 @@ all: @DEF_MAKE_ALL_RULE@ .PHONY: build_all build_all: check-clean-src $(BUILDPYTHON) $(BUILDPYTHONW) $(BUILDVENVLAUNCHER) $(BUILDVENVWLAUNCHER) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil $(ABI3DLLLIBRARY) $(ABI3LDLIBRARY) .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ @@ -808,6 +810,9 @@ pythonw_exe.o: $(srcdir)/PC/pythonw_exe.rc python_nt.o: $(srcdir)/PC/python_nt.rc $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ +python3dll_nt.o: $(srcdir)/PC/python_nt.rc + $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(ABI3DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ + venvlauncher.o: $(srcdir)/PC/pylauncher.rc $(WINDRES) $(RCFLAGS) -DPY_ICON -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pylauncher.rc $@ @@ -921,6 +926,10 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) python_nt.o else true; \ fi +$(ABI3DLLLIBRARY) $(ABI3LDLIBRARY): python3dll_nt.o $(srcdir)/PC/launcher.c + $(LDSHARED) -DPYTHON_DLL_NAME=\"$(DLLLIBRARY)\" $(srcdir)/PC/python3dll.c -Wl,--out-implib=$(ABI3LDLIBRARY) -o $(ABI3DLLLIBRARY) python3dll_nt.o \ + $(LDFLAGS_NODIST); + # wasm32-emscripten browser build # wasm assets directory is relative to current build dir, e.g. "./usr/local". # --preload-file turns a relative asset path into an absolute path. @@ -2029,6 +2038,7 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ if test -f $(LDLIBRARY) && test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ if test -n "$(DLLLIBRARY)" ; then \ $(INSTALL_SHARED) $(DLLLIBRARY) $(DESTDIR)$(BINDIR); \ + $(INSTALL_SHARED) $(ABI3DLLLIBRARY) $(DESTDIR)$(BINDIR); \ else \ $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(LIBDIR)/$(INSTSONAME); \ if test $(LDLIBRARY) != $(INSTSONAME); then \ @@ -2506,6 +2516,7 @@ libainstall: all scripts if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ if test "$(SHLIB_SUFFIX)" = .dll -o "$(SHLIB_SUFFIX)" = .pyd; then \ $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBDIR) ; \ + $(INSTALL_DATA) $(ABI3LDLIBRARY) $(DESTDIR)$(LIBDIR) ; \ else \ $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \ fi; \ @@ -2759,7 +2770,7 @@ clean: clean-retain-profile clean-bolt .PHONY: clobber clobber: clean - -rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ + -rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY)$(ABI3LDLIBRARY) $(ABI3DLLLIBRARY) \ tags TAGS \ config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform diff --git a/PC/python3dll.c b/PC/python3dll.c index 505feef4b986c4..ee9380c81437e8 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -2,6 +2,7 @@ /* Generated by Tools/build/stable_abi.py */ +#ifdef _MSC_VER #ifdef _M_IX86 #define DECORATE "_" #else @@ -12,6 +13,13 @@ __pragma(comment(linker, "/EXPORT:" DECORATE #name "=" PYTHON_DLL_NAME "." #name)) #define EXPORT_DATA(name) \ __pragma(comment(linker, "/EXPORT:" DECORATE #name "=" PYTHON_DLL_NAME "." #name ",DATA")) +#else +// XXX: Why do we need the .dll extension and no DECORATE compared to the MSVC case? +#define EXPORT_FUNC(name) \ + asm(".section .drectve\n\t.ascii \" -export:" #name "=\\\"" PYTHON_DLL_NAME "." #name "\\\" \""); +#define EXPORT_DATA(name) \ + asm(".section .drectve\n\t.ascii \" -export:" #name "=\\\"" PYTHON_DLL_NAME "." #name "\\\",DATA \""); +#endif EXPORT_FUNC(_Py_BuildValue_SizeT) EXPORT_FUNC(_Py_CheckRecursiveCall) From f72f52c9d19124e4f87b0023578719061a6ff079 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 27 Aug 2023 15:00:32 +0200 Subject: [PATCH 101/127] Port GetPythonImport() to mingw This looks for DLL names in the import table but while with MSVC the DLL is named python311.dll in our case it is named libpython3.11.dll. Adjust the strings and lengths accordingly. --- Python/dynload_win.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index db7968742b2228..36d1c7a0353968 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -56,6 +56,16 @@ const char *_PyImport_DynLoadFiletab[] = { #define DWORD_AT(mem) (*(DWORD *)(mem)) #define WORD_AT(mem) (*(WORD *)(mem)) +#ifdef __MINGW32__ +#define DLL_PREFIX "libpython" +#define DLL_PREFIX_LEN 9 +#define DLL_NAME_FORMAT "libpython%d.%d" +#else +#define DLL_PREFIX "python" +#define DLL_PREFIX_LEN 6 +#define DLL_NAME_FORMAT "python%d%d" +#endif + static char *GetPythonImport (HINSTANCE hModule) { unsigned char *dllbase, *import_data, *import_name; @@ -122,15 +132,15 @@ static char *GetPythonImport (HINSTANCE hModule) import_off); while (DWORD_AT(import_data)) { import_name = dllbase + DWORD_AT(import_data+12); - if (strlen(import_name) >= 6 && - !strncmp(import_name,"python",6)) { + if (strlen(import_name) >= DLL_PREFIX_LEN && + !strncmp(import_name, DLL_PREFIX, DLL_PREFIX_LEN)) { char *pch; /* Don't claim that python3.dll is a Python DLL. */ #ifdef _DEBUG - if (strcmp(import_name, "python3_d.dll") == 0) { + if (strcmp(import_name, DLL_PREFIX "3_d.dll") == 0) { #else - if (strcmp(import_name, "python3.dll") == 0) { + if (strcmp(import_name, DLL_PREFIX "3.dll") == 0) { #endif import_data += 20; continue; @@ -138,7 +148,7 @@ static char *GetPythonImport (HINSTANCE hModule) /* Ensure python prefix is followed only by numbers to the end of the basename */ - pch = import_name + 6; + pch = import_name + DLL_PREFIX_LEN; #ifdef _DEBUG while (*pch && pch[0] != '_' && pch[1] != 'd' && pch[2] != '.') { #else @@ -329,9 +339,9 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, PyOS_snprintf(buffer, sizeof(buffer), #ifdef _DEBUG - "python%d%d_d.dll", + DLL_NAME_FORMAT "_d.dll", #else - "python%d%d.dll", + DLL_NAME_FORMAT ".dll", #endif PY_MAJOR_VERSION,PY_MINOR_VERSION); import_python = GetPythonImport(hDLL); From 4538f351001dbed681093df085c068e2cff4d8f6 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 27 Aug 2023 16:04:50 +0200 Subject: [PATCH 102/127] LoadLibraryExW: make sure to only use backslashes for paths It seems like in case the path passed to it is absolute, but contains forward slashes then LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR does not work and DLLs in the same directory as the extension are not considered. This occurs in our fork because in MSYS2-mode the extension loader will normalize to forward slashes before. Normalize everything to backslashes again before passing it to LoadLibraryExW. Fixes #151 --- Python/dynload_win.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 36d1c7a0353968..caa49681d175e9 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -252,10 +252,22 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, _Py_CheckPython3(); #endif /* _MSC_VER */ +// So we can adjust the separators in the path below +#define USE_UNICODE_WCHAR_CACHE 0 + wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL); if (wpathname == NULL) return NULL; + // LoadLibraryExW only considers paths using backslashes as "fully qualified", + // and for example LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR doesn't work with forward slashes. + // https://github.com/msys2-contrib/cpython-mingw/issues/151 + for (size_t i = 0; wpathname[i] != L'\0'; ++i) { + if (wpathname[i] == L'/') { + wpathname[i] = L'\\'; + } + } + PyOS_snprintf(funcname, sizeof(funcname), "%.20s_%.200s", prefix, shortname); { From c7e9b9f4c5c6aa86598a2a5a845784bebdbbac11 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Wed, 8 Nov 2023 19:44:22 +0530 Subject: [PATCH 103/127] mingw_smoketests: add a test to check if sysconfig returns correct values for `LIBDIR` --- mingw_smoketests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index ca1f652caeec27..4f81c68803199c 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -218,6 +218,9 @@ def test_platform_things(self): import sysconfig import platform import importlib.machinery + import tempfile + import venv + import subprocess self.assertEqual(sys.implementation.name, "cpython") self.assertEqual(sys.platform, "win32") self.assertTrue(sysconfig.get_platform().startswith("mingw")) @@ -241,6 +244,21 @@ def test_platform_things(self): self.assertEqual(platform.system(), "Windows") self.assertTrue(isinstance(sys.api_version, int) and sys.api_version > 0) + with tempfile.TemporaryDirectory() as tmp: + builder = venv.EnvBuilder() + builder.create(tmp) + # This will not work in in-tree build + if not sysconfig.is_python_build(): + op = subprocess.check_output( + [ + os.path.join(tmp, "bin", "python.exe"), + "-c", + "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))" + ], + cwd=tmp, + ) + self.assertTrue(op.decode().strip().startswith(sys.base_prefix)) + def test_sys_getpath(self): # everything sourced from getpath.py import sys From 23c180a7c8baa6c73a50f9f3366705cfb84e397b Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 13:42:01 +0530 Subject: [PATCH 104/127] Add missing library for multiprocessing module `ws2_32` is required when building that module --- configure.ac | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 9671d724c52e51..0a82add8eb061a 100644 --- a/configure.ac +++ b/configure.ac @@ -4282,6 +4282,13 @@ AS_CASE([$MACHDEP], [SOCKET_LIBS=""] ) +dnl On MINGW, you need to link againt ws2_32 for _multiprocessing +AS_CASE([$MACHDEP], + [win32], [MULTIPROCESSING_LIBS="-lws2_32"], + [MULTIPROCESSING_LIBS=""] +) + + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -7899,7 +7906,7 @@ PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules PY_STDLIB_MOD([_multiprocessing], [], [test "$ac_cv_func_sem_unlink" = "yes" -o "$MACHDEP" = "win32"], - [-I\$(srcdir)/Modules/_multiprocessing]) + [-I\$(srcdir)/Modules/_multiprocessing], [$MULTIPROCESSING_LIBS]) PY_STDLIB_MOD([_posixshmem], [], [test "$have_posix_shmem" = "yes"], [$POSIXSHMEM_CFLAGS], [$POSIXSHMEM_LIBS]) From aaee576929e1baa4259c69a6655c1527dff49655 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 14:14:45 +0530 Subject: [PATCH 105/127] selectmodule: add support for MINGW --- Modules/selectmodule.c | 6 +++--- configure.ac | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 50788e5344c152..89396f31d0daf3 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -169,9 +169,9 @@ seq2set(PyObject *seq, fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) v = PyObject_AsFileDescriptor( o ); if (v == -1) goto finally; -#if defined(_MSC_VER) +#if defined(MS_WIN32) max = 0; /* not used for Win32 */ -#else /* !_MSC_VER */ +#else /* !MS_WIN32 */ if (!_PyIsSelectable_fd(v)) { PyErr_SetString(PyExc_ValueError, "filedescriptor out of range in select()"); @@ -179,7 +179,7 @@ seq2set(PyObject *seq, fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) } if (v > max) max = v; -#endif /* _MSC_VER */ +#endif /* MS_WIN32 */ FD_SET(v, set); /* add object and its file descriptor to the list */ diff --git a/configure.ac b/configure.ac index 0a82add8eb061a..353ff024d67e9f 100644 --- a/configure.ac +++ b/configure.ac @@ -4288,6 +4288,11 @@ AS_CASE([$MACHDEP], [MULTIPROCESSING_LIBS=""] ) +dnl On MINGW, you need to link againt ws2_32 for selectmodule +AS_CASE([$MACHDEP], + [win32], [SELECTMODULE_LIBS="-lws2_32"], + [SELECTMODULE_LIBS=""] +) dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -7896,7 +7901,7 @@ PY_STDLIB_MOD_SIMPLE([_opcode]) PY_STDLIB_MOD_SIMPLE([_pickle]) PY_STDLIB_MOD_SIMPLE([_queue]) PY_STDLIB_MOD_SIMPLE([_random]) -PY_STDLIB_MOD_SIMPLE([select]) +PY_STDLIB_MOD_SIMPLE([select], [], [$SELECTMODULE_LIBS]) PY_STDLIB_MOD_SIMPLE([_struct]) PY_STDLIB_MOD_SIMPLE([_typing]) PY_STDLIB_MOD_SIMPLE([_xxsubinterpreters]) From 31f7d79691b610e2043ae9dc583ffa23d86b749e Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 14:19:41 +0530 Subject: [PATCH 106/127] readline: disable on MINGW --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 353ff024d67e9f..e15e19af4ef66c 100644 --- a/configure.ac +++ b/configure.ac @@ -7760,6 +7760,7 @@ AS_CASE([$ac_sys_system], [VxWorks*], [PY_STDLIB_MOD_SET_NA([_scproxy], [_crypt], [termios], [grp])], [Darwin], [PY_STDLIB_MOD_SET_NA([ossaudiodev], [spwd])], [CYGWIN*], [PY_STDLIB_MOD_SET_NA([_scproxy], [nis])], + [MINGW*], [PY_STDLIB_MOD_SET_NA([readline])], [QNX*], [PY_STDLIB_MOD_SET_NA([_scproxy], [nis])], [FreeBSD*], [PY_STDLIB_MOD_SET_NA([_scproxy], [spwd])], [Emscripten|WASI], [ From aba641db02f42ee4dc11613788391515a5f97de7 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 14:25:11 +0530 Subject: [PATCH 107/127] Add libraries to fix compile of ctypes on MINGW --- configure.ac | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index e15e19af4ef66c..159d4feafab43c 100644 --- a/configure.ac +++ b/configure.ac @@ -4294,6 +4294,12 @@ AS_CASE([$MACHDEP], [SELECTMODULE_LIBS=""] ) +dnl On MINGW, you need to link againt ole32, oleaut32 and uuid for ctypes +AS_CASE([$MACHDEP], + [win32], [CTYPES_LIBS="-lole32 -loleaut32 -luuid"], + [CTYPES_LIBS=""] +) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -7991,7 +7997,7 @@ PY_STDLIB_MOD([_crypt], [$LIBCRYPT_CFLAGS], [$LIBCRYPT_LIBS]) PY_STDLIB_MOD([_ctypes], [], [test "$have_libffi" = yes], - [$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS], [$LIBFFI_LIBS]) + [$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS], [$LIBFFI_LIBS $CTYPES_LIBS]) PY_STDLIB_MOD([_curses], [], [test "$have_curses" != "no"], [$CURSES_CFLAGS], [$CURSES_LIBS] @@ -8062,7 +8068,7 @@ PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testconsole], [test "$TEST_MODULES" = yes -a "$MACHDEP" = "win32"]) PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$have_dynamic_loading" = yes], - [], [$LIBM]) + [], [$LIBM $CTYPES_LIBS]) dnl Limited API template modules. dnl The limited C API is not compatible with the Py_TRACE_REFS macro. From f95f4f4f0be70909eef94a07890681be00b061be Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 14:48:28 +0530 Subject: [PATCH 108/127] Add `-lpython` while linking stdlib Generalize the `MINGW` case. In theory, it should be correct for other system too and would fix cross-compilation --- Modules/makesetup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/makesetup b/Modules/makesetup index f000c9cd67310e..5e2ada098a5452 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -88,7 +88,7 @@ NL='\ ' # Setup to link with extra libraries when making shared extensions. -# Currently, only Cygwin needs this baggage. +# Currently, only Cygwin and MINGW needs this baggage. case `uname -s` in CYGWIN*) if test $libdir = . then @@ -97,6 +97,9 @@ CYGWIN*) if test $libdir = . ExtraLibDir='$(LIBPL)' fi ExtraLibs="-L$ExtraLibDir -lpython\$(LDVERSION)";; +*) + ExtraLibs='$(BLDLIBRARY)' + ExtraLibDepends='$(LIBRARY_DEPS)';; esac # Main loop @@ -285,7 +288,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | BUILT_SHARED="$BUILT_SHARED $mod" ;; esac - rule="$file: $objs" + rule="$file: $objs $ExtraLibDepends" rule="$rule; \$(BLDSHARED) $objs $libs $ExtraLibs -o $file" echo "$rule" >>$rulesf done From b2eab751414b439d440d3d29ea4c4fd2dc29984a Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 15:04:43 +0530 Subject: [PATCH 109/127] Enable `_uuid` on MINGW --- configure.ac | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 159d4feafab43c..2575149cfc8956 100644 --- a/configure.ac +++ b/configure.ac @@ -4300,6 +4300,12 @@ AS_CASE([$MACHDEP], [CTYPES_LIBS=""] ) +dnl On MINGW, you need to link againt rpcrt4 for _uuid +AS_CASE([$MACHDEP], + [win32], [UUID_LIBS="-lrpcrt4"], + [UUID_LIBS=""] +) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -8027,8 +8033,8 @@ PY_STDLIB_MOD([_tkinter], [], [test "$have_tcltk" = "yes"], [$TCLTK_CFLAGS], [$TCLTK_LIBS]) PY_STDLIB_MOD([_uuid], - [], [test "$have_uuid" = "yes"], - [$LIBUUID_CFLAGS], [$LIBUUID_LIBS]) + [], [test "$have_uuid" = "yes" -o "$MACHDEP" = "win32"], + [$LIBUUID_CFLAGS], [$LIBUUID_LIBS $UUID_LIBS]) dnl compression libs PY_STDLIB_MOD([zlib], [], [test "$have_zlib" = yes], From dceead90dec13e4bb730f28e300b891e60102793 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 1 Jan 2024 16:43:41 +0530 Subject: [PATCH 110/127] Enable curses library on MINGW --- configure.ac | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configure.ac b/configure.ac index 2575149cfc8956..c4dfe3b46825da 100644 --- a/configure.ac +++ b/configure.ac @@ -6868,6 +6868,14 @@ dnl have_panel=[no, panelw, panel] have_curses=no have_panel=no +case "$host_os" in + mingw*) + # On Mingw, include directory is different, so just assume + # it's there. + ac_cv_header_ncurses_h=yes + ;; +esac + AH_TEMPLATE([HAVE_NCURSESW], [Define to 1 if you have the `ncursesw' library.]) AC_CHECK_HEADERS([curses.h ncurses.h]) From 62ef1e650f63f53894c5acb55ab8828fff8a0c4c Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 2 Feb 2024 14:43:29 +0530 Subject: [PATCH 111/127] mingw_smoketests: disable distutils related tests --- mingw_smoketests.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 4f81c68803199c..0ab5d3b60a443e 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -61,24 +61,24 @@ def test_userdir_path_sep(self): self.assertEqual(str(Path("~").expanduser()), expanduser("~")) self.assertEqual(str(Path.home()), expanduser("~")) - def test_sysconfig_schemes(self): - # https://github.com/msys2/MINGW-packages/issues/9319 - import sysconfig - from distutils.dist import Distribution - from distutils.command.install import install - - names = ['scripts', 'purelib', 'platlib', 'data', 'include'] - for scheme in ["nt", "nt_user"]: - for name in names: - c = install(Distribution({"name": "foobar"})) - c.user = (scheme == "nt_user") - c.finalize_options() - if name == "include": - dist_path = os.path.dirname(getattr(c, "install_" + "headers")) - else: - dist_path = getattr(c, "install_" + name) - sys_path = sysconfig.get_path(name, scheme) - self.assertEqual(dist_path, sys_path, (scheme, name)) + # def test_sysconfig_schemes(self): + # # https://github.com/msys2/MINGW-packages/issues/9319 + # import sysconfig + # from distutils.dist import Distribution + # from distutils.command.install import install + + # names = ['scripts', 'purelib', 'platlib', 'data', 'include'] + # for scheme in ["nt", "nt_user"]: + # for name in names: + # c = install(Distribution({"name": "foobar"})) + # c.user = (scheme == "nt_user") + # c.finalize_options() + # if name == "include": + # dist_path = os.path.dirname(getattr(c, "install_" + "headers")) + # else: + # dist_path = getattr(c, "install_" + name) + # sys_path = sysconfig.get_path(name, scheme) + # self.assertEqual(dist_path, sys_path, (scheme, name)) def test_ctypes_find_library(self): from ctypes.util import find_library From 9e56012dda86ade3274903360ecf5847b3d178d4 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Mon, 19 Feb 2024 22:09:25 +0530 Subject: [PATCH 112/127] configure.ac: use autoconf 2.69 llvm-mingw only has autoconf 2.69 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c4dfe3b46825da..fe4d39fc879a97 100644 --- a/configure.ac +++ b/configure.ac @@ -12,7 +12,7 @@ dnl # Set VERSION so we only need to edit in one place (i.e., here) m4_define([PYTHON_VERSION], [3.12]) -AC_PREREQ([2.71]) +AC_PREREQ([2.69]) AC_INIT([python],[PYTHON_VERSION],[https://github.com/python/cpython/issues/]) From b67e3c67e65e22bf2ee8cf2d2f11b720dfca012e Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Feb 2024 11:04:39 +0530 Subject: [PATCH 113/127] CI: clean up ignored tests Remove tests which are passing now, or no longer exist, and move tests which only fail with msvcrt into a separate file. --- .github/workflows/mingw.yml | 12 ++++++++++-- Lib/test/test_wmi.py | 2 +- mingw_ignorefile.txt | 24 +++--------------------- mingw_ignorefile_msvcrt.txt | 8 ++++++++ 4 files changed, 22 insertions(+), 24 deletions(-) create mode 100644 mingw_ignorefile_msvcrt.txt diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 0c9f644d84cca5..827dfe5a3705e5 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -89,16 +89,24 @@ jobs: shell: msys2 {0} run: | IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + IGNOREFILE_EXTRA="$IGNOREFILE" + if [[ "${{ matrix.msystem }}" == "MINGW32" ]] || [[ "${{ matrix.msystem }}" == "MINGW64" ]]; then + IGNOREFILE_EXTRA="$(pwd)/mingw_ignorefile_msvcrt.txt" + fi cd _build - MSYSTEM= ./python.exe -m test -j8 --ignorefile "$IGNOREFILE" -W + MSYSTEM= ./python.exe -m test -j8 --ignorefile "$IGNOREFILE" --ignorefile "$IGNOREFILE_EXTRA" -W - name: Run broken tests continue-on-error: true shell: msys2 {0} run: | IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + IGNOREFILE_EXTRA="$IGNOREFILE" + if [[ "${{ matrix.msystem }}" == "MINGW32" ]] || [[ "${{ matrix.msystem }}" == "MINGW64" ]]; then + IGNOREFILE_EXTRA="$(pwd)/mingw_ignorefile_msvcrt.txt" + fi cd _build - MSYSTEM= ./python.exe -m test -j8 --matchfile "$IGNOREFILE" -W + MSYSTEM= ./python.exe -m test -j8 --matchfile "$IGNOREFILE" --matchfile "$IGNOREFILE_EXTRA" -W - name: Install shell: msys2 {0} diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index 111f990656ced5..3dc452e79fe388 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -8,7 +8,7 @@ # Do this first so test will be skipped if module doesn't exist -_wmi = import_helper.import_module('_wmi', required_on=['win']) +_wmi = import_helper.import_module('_wmi') def wmi_exec_query(query): diff --git a/mingw_ignorefile.txt b/mingw_ignorefile.txt index e692d7ff26eb4d..550e72413a4f4e 100644 --- a/mingw_ignorefile.txt +++ b/mingw_ignorefile.txt @@ -2,16 +2,11 @@ test.test_ctypes.test_loading.LoaderTest.test_load_dll_with_flags distutils.tests.test_bdist_dumb.BuildDumbTestCase.test_simple_built distutils.tests.test_cygwinccompiler.CygwinCCompilerTestCase.test_get_versions distutils.tests.test_util.UtilTestCase.test_change_root -test.datetimetester.TestLocalTimeDisambiguation_Fast.* -test.datetimetester.TestLocalTimeDisambiguation_Pure.* -test.test_cmath.CMathTests.test_specific_values -test.test_cmd_line_script.CmdLineTest.test_consistent_sys_path_for_direct_execution test.test_compileall.CommandLineTestsNoSourceEpoch.* test.test_compileall.CommandLineTestsWithSourceEpoch.* test.test_compileall.CompileallTestsWithoutSourceEpoch.* test.test_compileall.CompileallTestsWithSourceEpoch.* test.test_import.ImportTests.test_dll_dependency_import -test.test_math.MathTests.* test.test_ntpath.NtCommonTest.test_import test.test_os.StatAttributeTests.test_stat_block_device test.test_os.TestScandir.test_attributes @@ -23,21 +18,8 @@ test.test_site._pthFileTests.* test.test_site.HelperFunctionsTests.* test.test_site.StartupImportTests.* test.test_ssl.* -test.test_strptime.CalculationTests.* -test.test_strptime.StrptimeTests.test_weekday -test.test_strptime.TimeRETests.test_compile -test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date test.test_venv.BasicTest.* test.test_venv.EnsurePipTest.* -test.test_sysconfig.TestSysConfig.test_user_similar -test.test_tcl.TclTest.testLoadWithUNC -test.test_wmi -# flaky -test.test__xxsubinterpreters.* -test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe -test.test_asynchat.TestAsynchat.test_line_terminator2 -test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_gc_aclose_09 -test.test_concurrent_futures.ThreadPoolShutdownTest.test_interpreter_shutdown -test.test_asynchat.TestNotConnected.test_disallow_negative_terminator -test.test_logging.SysLogHandlerTest.* -test.test_logging.IPv6SysLogHandlerTest.* + +test.test_dict.DictTest.test_splittable_to_generic_combinedtable +test.test_capi.test_run.CAPITest.test_run_fileexflags diff --git a/mingw_ignorefile_msvcrt.txt b/mingw_ignorefile_msvcrt.txt new file mode 100644 index 00000000000000..be363bd0faf404 --- /dev/null +++ b/mingw_ignorefile_msvcrt.txt @@ -0,0 +1,8 @@ +test.datetimetester.TestLocalTimeDisambiguation_Fast.* +test.datetimetester.TestLocalTimeDisambiguation_Pure.* +test.test_cmath.CMathTests.test_specific_values +test.test_math.MathTests.* +test.test_strptime.CalculationTests.* +test.test_strptime.StrptimeTests.test_weekday +test.test_strptime.TimeRETests.test_compile +test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date From 9638204b854adedbeca465d1dc9f8e8a8b27d443 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Thu, 11 Jul 2024 13:58:20 +0530 Subject: [PATCH 114/127] gcc: make `incompatible-pointer-types` as warning gcc 14 changed `incompatible-pointer-types` from warning to error for now, make it as warning to avoid build failure --- configure.ac | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configure.ac b/configure.ac index fe4d39fc879a97..5d1b6e25269823 100644 --- a/configure.ac +++ b/configure.ac @@ -2528,6 +2528,14 @@ case $GCC in yes) CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" + # GCC 14 and later error out with incompatible-pointer-types + # this only occues on mingw + case $host in + *-*-mingw*) + CFLAGS_NODIST="$CFLAGS_NODIST -Wno-error=incompatible-pointer-types" + ;; + esac + PY_CHECK_CC_WARNING([enable], [extra], [if we can add -Wextra]) AS_VAR_IF([ac_cv_enable_extra_warning], [yes], [CFLAGS_NODIST="$CFLAGS_NODIST -Wextra"]) From 613d98133c785e4ff8fcae249e51956df7da93bf Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sun, 8 Sep 2024 15:04:42 +0530 Subject: [PATCH 115/127] test_makefile: normalize path so that it passes on mingw --- Lib/test/test_tools/test_makefile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index f1f0cd87ba7496..596e50dfbe2842 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -33,11 +33,12 @@ def list_test_dirs(self): if '\t' not in line: break result.append(line.replace('\\', '').strip()) + result = [os.path.normpath(d) for d in result if d] return result def test_makefile_test_folders(self): test_dirs = self.list_test_dirs() - idle_test = 'idlelib/idle_test' + idle_test = os.path.join('idlelib', 'idle_test') self.assertIn(idle_test, test_dirs) used = [idle_test] From c4e2de6cd7e3e139cfc5016256970e876b89a1c9 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Thu, 12 Sep 2024 17:42:31 +0530 Subject: [PATCH 116/127] `sysconfig.get_platform()`: use consistent naming Currently, the platform names are hardcoded in many places, and are not named consistently. This commit fixes it by standardizing the names to be returned by `sysconfig.get_platform`. The naming is based on https://github.com/msys2-contrib/cpython-mingw/issues/167#issuecomment-1952215164 Similarly, `EXT_SUFFIX` is also standardized to be consistent with the platform names. Signed-off-by: Naveen M K --- Lib/site.py | 32 +++++++++------ Lib/sysconfig.py | 32 +++++++++------ Lib/test/test_importlib/test_windows.py | 38 +++++++++-------- configure.ac | 54 ++++++++++--------------- mingw_smoketests.py | 2 +- 5 files changed, 83 insertions(+), 75 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index c1ecd6ec4739c3..16c7c41cd3d0c6 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -314,21 +314,27 @@ def joinuser(*args): def _get_platform(): if os.name == 'nt': if 'gcc' in sys.version.lower(): + platform = 'mingw' + if 'amd64' in sys.version.lower(): + platform += '_x86_64' + elif 'arm64' in sys.version.lower(): + platform += '_aarch64' + elif 'arm' in sys.version.lower(): + platform += '_armv7' + else: + platform += '_i686' + if 'ucrt' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_ucrt' - return 'mingw_i686_ucrt' + platform += '_ucrt' + else: + platform += "_msvcrt" + if 'clang' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_clang' - if 'arm64' in sys.version.lower(): - return 'mingw_aarch64' - if 'arm' in sys.version.lower(): - return 'mingw_armv7' - return 'mingw_i686_clang' - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64' - return 'mingw_i686' + platform += "_llvm" + else: + platform += "_gnu" + + return platform return sys.platform # Same to sysconfig.get_path('purelib', os.name+'_user') diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 916c1e82b2d9b7..c12ce38dec5b3d 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -818,21 +818,27 @@ def get_platform(): """ if os.name == 'nt': if 'gcc' in sys.version.lower(): + platform = 'mingw' + if 'amd64' in sys.version.lower(): + platform += '_x86_64' + elif 'arm64' in sys.version.lower(): + platform += '_aarch64' + elif 'arm' in sys.version.lower(): + platform += '_armv7' + else: + platform += '_i686' + if 'ucrt' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_ucrt' - return 'mingw_i686_ucrt' + platform += '_ucrt' + else: + platform += "_msvcrt" + if 'clang' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_clang' - if 'arm64' in sys.version.lower(): - return 'mingw_aarch64' - if 'arm' in sys.version.lower(): - return 'mingw_armv7' - return 'mingw_i686_clang' - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64' - return 'mingw_i686' + platform += "_llvm" + else: + platform += "_gnu" + + return platform if 'amd64' in sys.version.lower(): return 'win-amd64' if '(arm)' in sys.version.lower(): diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index 03b5a98974f3fa..0b1254c2d75896 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -25,22 +25,28 @@ def get_platform(): 'arm' : 'win-arm32', } if os.name == 'nt': - if 'gcc' in sys.version.lower(): - if 'ucrt' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_ucrt' - return 'mingw_i686_ucrt' - if 'clang' in sys.version.lower(): - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64_clang' - if 'arm64' in sys.version.lower(): - return 'mingw_aarch64' - if 'arm' in sys.version.lower(): - return 'mingw_armv7' - return 'mingw_i686_clang' - if 'amd64' in sys.version.lower(): - return 'mingw_x86_64' - return 'mingw_i686' + if "gcc" in sys.version.lower(): + platform = "mingw" + if "amd64" in sys.version.lower(): + platform += "_x86_64" + elif "arm64" in sys.version.lower(): + platform += "_aarch64" + elif "arm" in sys.version.lower(): + platform += "_armv7" + else: + platform += "_i686" + + if "ucrt" in sys.version.lower(): + platform += "_ucrt" + else: + platform += "_msvcrt" + + if "clang" in sys.version.lower(): + platform += "_llvm" + else: + platform += "_gnu" + + return platform if ('VSCMD_ARG_TGT_ARCH' in os.environ and os.environ['VSCMD_ARG_TGT_ARCH'] in TARGET_TO_PLAT): return TARGET_TO_PLAT[os.environ['VSCMD_ARG_TGT_ARCH']] diff --git a/configure.ac b/configure.ac index 5d1b6e25269823..264bae613ef66a 100644 --- a/configure.ac +++ b/configure.ac @@ -6335,16 +6335,12 @@ AC_C_BIGENDIAN AC_SUBST(PYD_PLATFORM_TAG) # Special case of PYD_PLATFORM_TAG with python build with mingw. -# Python can with compiled with clang or gcc and linked -# to msvcrt or ucrt. To avoid conflicts between them -# we are selecting the extension as based on the compiler -# and the runtime they link to -# gcc + x86_64 + msvcrt = cp{version number}-x86_64 -# gcc + i686 + msvcrt = cp{version number}-i686 -# gcc + x86_64 + ucrt = cp{version number}-x86_64-ucrt -# clang + x86_64 + ucrt = cp{version number}-x86_64-clang -# clang + i686 + ucrt = cp{version number}-i686-clang - +# Python can with different cpu arch and c runtime as well as different +# toolchain. We follow this`mingw___` +# convention for PYD_PLATFORM_TAG. Where: +# `cpu_arch` = `x86_64`, `aarch64` or `i686` +# `c_runtime` = `msvcrt` or `ucrt` +# `toolchain` = `gnu` or `llvm` PYD_PLATFORM_TAG="" case $host in *-*-mingw*) @@ -6363,38 +6359,32 @@ esac case $host_os in mingw*) AC_MSG_CHECKING(PYD_PLATFORM_TAG) + PYD_PLATFORM_TAG="mingw" case $host in i686-*-mingw*) - if test -n "${cc_is_clang}"; then - # it is CLANG32 - PYD_PLATFORM_TAG="mingw_i686_clang" - else - if test $linking_to_ucrt = no; then - PYD_PLATFORM_TAG="mingw_i686" - else - PYD_PLATFORM_TAG="mingw_i686_ucrt" - fi - fi + PYD_PLATFORM_TAG+="_i686" ;; x86_64-*-mingw*) - if test -n "${cc_is_clang}"; then - # it is CLANG64 - PYD_PLATFORM_TAG="mingw_x86_64_clang" - else - if test $linking_to_ucrt = no; then - PYD_PLATFORM_TAG="mingw_x86_64" - else - PYD_PLATFORM_TAG="mingw_x86_64_ucrt" - fi - fi + PYD_PLATFORM_TAG+="_x86_64" ;; aarch64-*-mingw*) - PYD_PLATFORM_TAG+="mingw_aarch64" + PYD_PLATFORM_TAG+="_aarch64" ;; armv7-*-mingw*) - PYD_PLATFORM_TAG+="mingw_armv7" + PYD_PLATFORM_TAG+="_armv7" ;; esac + if test $linking_to_ucrt = no; then + PYD_PLATFORM_TAG+="_msvcrt" + else + PYD_PLATFORM_TAG += "_ucrt" + fi + if test -n "${cc_is_clang}"; then + # it is CLANG32 + PYD_PLATFORM_TAG+="_llvm" + else + PYD_PLATFORM_TAG+="_gnu" + fi AC_MSG_RESULT($PYD_PLATFORM_TAG) esac diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 0ab5d3b60a443e..9ad32b3a126db9 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -37,7 +37,7 @@ if sysconfig.is_python_build(): os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" -_UCRT = sysconfig.get_platform() not in ('mingw_x86_64', 'mingw_i686') +_UCRT = 'ucrt' in sysconfig.get_platform() class Tests(unittest.TestCase): From cc8c05c2720381facce6e07132a801a513971c9d Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 1 Nov 2024 11:47:01 +0530 Subject: [PATCH 117/127] Add ucrt to version string --- Python/getcompiler.c | 5 +++++ configure.ac | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Python/getcompiler.c b/Python/getcompiler.c index 4b0b9b328ba2f8..dd4d3c8f101468 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -32,8 +32,13 @@ #if defined(__clang__) #define str(x) #x #define xstr(x) str(x) +#if defined(_UCRT) +#define COMPILER COMP_SEP "[GCC UCRT Clang " xstr(__clang_major__) "." \ + xstr(__clang_minor__) "." xstr(__clang_patchlevel__) ARCH_SUFFIX "]" +#else #define COMPILER COMP_SEP "[GCC Clang " xstr(__clang_major__) "." \ xstr(__clang_minor__) "." xstr(__clang_patchlevel__) ARCH_SUFFIX "]" +#endif #else #if defined(_UCRT) #define COMPILER COMP_SEP "[GCC UCRT " __VERSION__ ARCH_SUFFIX "]" diff --git a/configure.ac b/configure.ac index 264bae613ef66a..10a6e8464f9add 100644 --- a/configure.ac +++ b/configure.ac @@ -6377,7 +6377,7 @@ case $host_os in if test $linking_to_ucrt = no; then PYD_PLATFORM_TAG+="_msvcrt" else - PYD_PLATFORM_TAG += "_ucrt" + PYD_PLATFORM_TAG+="_ucrt" fi if test -n "${cc_is_clang}"; then # it is CLANG32 From dee2b16fa1a2713ee87c50cd18d432e54b460489 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sat, 2 Nov 2024 11:30:54 +0530 Subject: [PATCH 118/127] socketmodule: fix captilization of headers `Rpc` should be `rpc`. It error's out in cross-compilation. --- Modules/socketmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 7038e10d663e41..f0d84d350b6ea8 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -279,7 +279,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # endif /* Helpers needed for AF_HYPERV */ -# include +# include /* Macros based on the IPPROTO enum, see: https://bugs.python.org/issue29515 */ #ifdef _MSC_VER From b86949a91b27a03461e991c4606084691bf254c3 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 22 Dec 2024 13:45:24 +0100 Subject: [PATCH 119/127] mingw_smoketests: build extension in a venv ensurepip no longer works since we implement PEP 668, so do everything in a venv. --- mingw_smoketests.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/mingw_smoketests.py b/mingw_smoketests.py index 9ad32b3a126db9..5af27645e5df7e 100644 --- a/mingw_smoketests.py +++ b/mingw_smoketests.py @@ -287,10 +287,14 @@ def test_c_ext_build(self): import sys import subprocess import textwrap + import venv from pathlib import Path with tempfile.TemporaryDirectory() as tmppro: - subprocess.check_call([sys.executable, "-m", "ensurepip", "--user"]) + builder = venv.EnvBuilder(with_pip=True) + builder.create(tmppro) + venv_exe = os.path.join(tmppro, "bin", os.path.basename(sys.executable)) + with Path(tmppro, "setup.py").open("w") as f: f.write( textwrap.dedent( @@ -342,20 +346,11 @@ def test_c_ext_build(self): ) ) subprocess.check_call( - [sys.executable, "-c", "import struct"], - ) - subprocess.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - "wheel", - ], + [venv_exe, "-c", "import struct"], ) subprocess.check_call( [ - sys.executable, + venv_exe, "-m", "pip", "install", @@ -363,7 +358,7 @@ def test_c_ext_build(self): ], ) subprocess.check_call( - [sys.executable, "-c", "import cwrapper"], + [venv_exe, "-c", "import cwrapper"], ) From 963fcc79deb13b8ef3aefc42eef8e18f6897f41c Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Mon, 23 Dec 2024 06:03:11 +0100 Subject: [PATCH 120/127] venvlauncher: try looking for the versioned .exe first and then fall back By default venvlauncher only looks for python.exe in the python "home", which fails in the MSYS2 case where there could be multiple python versions in the same prefix i.e. python.exe could be 3.12 while the venv was created with python3.13.exe. Upstream CPython doesn't have this problem since they never have multiple Python versions in the same prefix. On the other hand the Python test suite (test_sysconfig specifically) creates a venv for the uninstalled Python build dir, where only python.exe exists. To cover both cases we first try to look for the python3.XY.exe and then fall back to python.exe. --- Makefile.pre.in | 4 ++-- PC/launcher.c | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index e9ded19a3fb519..68fbe7b1e7d9ec 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -827,10 +827,10 @@ $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) python_exe.o $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) python_exe.o $(BUILDVENVLAUNCHER): $(BUILDPYTHON) venvlauncher.o $(srcdir)/PC/launcher.c - $(LINKCC) -D_CONSOLE -DVENV_REDIRECT $(PY_STDMODULE_CFLAGS) -municode -static -static-libgcc -static-libstdc++ venvlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion + $(LINKCC) -D_CONSOLE -DVENV_REDIRECT -DPYTHON_EXECUTABLE_WITH_VERSION="L\"python$(LDVERSION)$(EXE)\"" $(PY_STDMODULE_CFLAGS) -municode -static -static-libgcc -static-libstdc++ venvlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion $(BUILDVENVWLAUNCHER): $(BUILDPYTHONW) venvwlauncher.o $(srcdir)/PC/launcher.c - $(LINKCC) -D_WINDOWS -DVENV_REDIRECT $(PY_STDMODULE_CFLAGS) -mwindows -municode -static -static-libgcc -static-libstdc++ venvwlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion + $(LINKCC) -D_WINDOWS -DVENV_REDIRECT -DPYTHON_EXECUTABLE_WITH_VERSION="L\"pythonw$(LDVERSION)$(EXE)\"" $(PY_STDMODULE_CFLAGS) -mwindows -municode -static -static-libgcc -static-libstdc++ venvwlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform diff --git a/PC/launcher.c b/PC/launcher.c index 14d8b8fff83d0c..ee7600468f1c8b 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1930,7 +1930,8 @@ process(int argc, wchar_t ** argv) if (!cch) { error(0, L"Cannot determine memory for home path"); } - cch += (DWORD)wcslen(PYTHON_EXECUTABLE) + 4; /* include sep, null and quotes */ + cch += (DWORD)max(wcslen(PYTHON_EXECUTABLE_WITH_VERSION), + wcslen(PYTHON_EXECUTABLE)) + 4; /* include sep, null and quotes */ executable = (wchar_t *)malloc(cch * sizeof(wchar_t)); if (executable == NULL) { error(RC_NO_MEMORY, L"A memory allocation failed"); @@ -1948,13 +1949,22 @@ process(int argc, wchar_t ** argv) executable[cch_actual++] = L'\\'; executable[cch_actual] = L'\0'; } - if (wcscat_s(&executable[1], cch - 1, PYTHON_EXECUTABLE)) { + if (wcscat_s(&executable[1], cch - 1, PYTHON_EXECUTABLE_WITH_VERSION)) { error(RC_BAD_VENV_CFG, L"Cannot create executable path from '%ls'", venv_cfg_path); } /* there's no trailing quote, so we only have to skip one character for the test */ + // Check if the versioned executable (PYTHON_EXECUTABLE_WITH_VERSION) exists first if (GetFileAttributesW(&executable[1]) == INVALID_FILE_ATTRIBUTES) { - error(RC_NO_PYTHON, L"No Python at '%ls'", executable); + // If not found, try PYTHON_EXECUTABLE + executable[cch_actual] = L'\0'; // Reset the path + if (wcscat_s(&executable[1], cch - 1, PYTHON_EXECUTABLE)) { + error(RC_BAD_VENV_CFG, L"Cannot create executable path from '%ls'", + venv_cfg_path); + } + if (GetFileAttributesW(&executable[1]) == INVALID_FILE_ATTRIBUTES) { + error(RC_NO_PYTHON, L"No Python at '%ls'", executable); + } } /* now append the final quote */ wcscat_s(executable, cch, L"\""); From 9aec508e729cbdcd3eaed874bf73c349093076d9 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Thu, 6 Feb 2025 11:12:20 +0100 Subject: [PATCH 121/127] CI: update actions --- .github/workflows/mingw.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 827dfe5a3705e5..65d7d7decfc524 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -22,7 +22,7 @@ jobs: run: | git config --global core.autocrlf false git config --global core.eol lf - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: msys2/setup-msys2@v2 with: msystem: ${{ matrix.msystem }} @@ -150,7 +150,7 @@ jobs: tar -zcf python.tar.gz python_pkgdir/ - name: Upload - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: build-${{ matrix.msystem }} @@ -161,7 +161,7 @@ jobs: container: image: ubuntu:24.10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install deps run: | apt-get update @@ -175,7 +175,7 @@ jobs: make \ pkg-config - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.12' @@ -207,7 +207,7 @@ jobs: zip -r install.zip _build/install - name: Upload - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: build-cross-gcc-x86_64 path: install.zip @@ -216,7 +216,7 @@ jobs: needs: [cross-gcc-x86_64] runs-on: windows-latest steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-cross-gcc-x86_64 @@ -235,7 +235,7 @@ jobs: matrix: arch: ['x86_64', 'i686', 'aarch64', 'armv7'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install deps run: | @@ -272,7 +272,7 @@ jobs: zip -r install.zip _build/install - name: Upload - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: build-cross-llvm-mingw-${{ matrix.arch }} path: install.zip @@ -285,7 +285,7 @@ jobs: matrix: arch: ['x86_64', 'i686'] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-cross-llvm-mingw-${{ matrix.arch }} From 56fd36b36cbb6e94190a8277df219bf5965673d8 Mon Sep 17 00:00:00 2001 From: jeremyd2019 Date: Sun, 23 Jan 2022 16:38:18 -0800 Subject: [PATCH 122/127] CI: test CLANGARM64 test cross-compiled arm builds too --- .github/workflows/mingw.yml | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 65d7d7decfc524..97ecd188aa6d28 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -3,20 +3,27 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: windows-2022 + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - msystem: ['MINGW64','MINGW32','UCRT64','CLANG64'] + msystem: ['MINGW64','MINGW32','UCRT64','CLANG64','CLANGARM64'] include: - msystem: MINGW64 prefix: mingw-w64-x86_64 + runner: windows-2022 - msystem: MINGW32 prefix: mingw-w64-i686 + runner: windows-2022 - msystem: UCRT64 prefix: mingw-w64-ucrt-x86_64 + runner: windows-2022 - msystem: CLANG64 prefix: mingw-w64-clang-x86_64 + runner: windows-2022 + - msystem: CLANGARM64 + prefix: mingw-w64-clang-aarch64 + runner: windows-11-arm steps: - name: Setup git run: | @@ -26,7 +33,7 @@ jobs: - uses: msys2/setup-msys2@v2 with: msystem: ${{ matrix.msystem }} - release: false + release: ${{ matrix.msystem == 'CLANGARM64' }} update: true install: >- make @@ -279,11 +286,22 @@ jobs: cross-llvm-mingw-test: needs: [cross-llvm-mingw] - runs-on: windows-latest + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - arch: ['x86_64', 'i686'] + arch: ['x86_64', 'i686', 'aarch64', 'armv7'] + include: + - arch: x86_64 + runner: windows-latest + - arch: i686 + runner: windows-latest + - arch: aarch64 + runner: windows-11-arm + # we can test this for now, but Windows 11 24H2 dropped support for + # arm32 + - arch: armv7 + runner: windows-11-arm steps: - uses: actions/download-artifact@v4 with: @@ -291,7 +309,9 @@ jobs: - name: 'Run tests' run: | - 7z x install.zip + if (Test-Path ./_build) {rm -r -fo ./_build} + Add-Type -AssemblyName System.IO.Compression.FileSystem ; + [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\install.zip", "$PWD") ./_build/install/usr/local/bin/python3.exe -c "import sysconfig, pprint; pprint.pprint(sysconfig.get_config_vars())" From e15c2cc1ec9d10c594b9b27c9d1b8e4b8073702b Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 6 Jun 2025 21:32:42 +0200 Subject: [PATCH 123/127] CI: disable pkg-config to avoid it finding build machine packages --- .github/workflows/mingw.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 97ecd188aa6d28..360218a39f1439 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -197,6 +197,9 @@ jobs: mkdir _build && cd _build + export PKG_CONFIG_LIBDIR=/dev/null + unset PKG_CONFIG_PATH + ../configure \ --host=x86_64-w64-mingw32 \ --build=x86_64-pc-linux-gnu \ @@ -259,6 +262,9 @@ jobs: mkdir _build && cd _build + export PKG_CONFIG_LIBDIR=/dev/null + unset PKG_CONFIG_PATH + export CC=${{ matrix.arch }}-w64-mingw32-clang export CXX=${CC}++ ../configure \ From 9ea3a72c3dc5156fdb4a63c08764b4f6115b4547 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Mon, 18 Aug 2025 20:48:29 +0200 Subject: [PATCH 124/127] CI: stop testing armv7, no longer possible The GHA provided runner now runs a newer Windows version which no longer allows running armv7 binaries, so stop testing them there. Same as with https://github.com/mingw-w64/mingw-w64/commit/e6535ce604431e885 --- .github/workflows/mingw.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 360218a39f1439..b0bf5df4dd61dc 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -296,7 +296,7 @@ jobs: strategy: fail-fast: false matrix: - arch: ['x86_64', 'i686', 'aarch64', 'armv7'] + arch: ['x86_64', 'i686', 'aarch64'] include: - arch: x86_64 runner: windows-latest @@ -304,10 +304,6 @@ jobs: runner: windows-latest - arch: aarch64 runner: windows-11-arm - # we can test this for now, but Windows 11 24H2 dropped support for - # arm32 - - arch: armv7 - runner: windows-11-arm steps: - uses: actions/download-artifact@v4 with: From 9e124264a09cb49751668212ded725713648f6d7 Mon Sep 17 00:00:00 2001 From: oltolm Date: Sun, 17 Aug 2025 12:37:07 +0200 Subject: [PATCH 125/127] venv: install batch scripts --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 68fbe7b1e7d9ec..db789988936dc1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2174,7 +2174,7 @@ LIBSUBDIRS= asyncio \ msilib \ unittest \ urllib \ - venv venv/scripts venv/scripts/common venv/scripts/posix \ + venv venv/scripts venv/scripts/common venv/scripts/nt venv/scripts/posix \ wsgiref \ $(XMLLIBSUBDIRS) \ xmlrpc \ From 5f118e451982c9502876e07fdf384d94dfff3975 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 31 Aug 2025 16:40:48 +0200 Subject: [PATCH 126/127] winreg: patch cleanup * REG_LEGAL_CHANGE_FILTER is part of mingw-w64 now * SIZEOF_HKEY is defined already * use the same build logic as with other win32 only modules Needs to be merged into "Build winreg by default when compiling for MINGW" Fixes #181 --- Modules/Setup.bootstrap.in | 2 +- PC/winreg.c | 19 ------------------- configure.ac | 8 +------- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index cec80f5a9f0f19..271a5b6ddc0266 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -36,6 +36,6 @@ _symtable symtablemodule.c @MODULE_PWD_TRUE@pwd pwdmodule.c # build-in modules for windows platform: -@USE_WIN32_MODULE@winreg ../PC/winreg.c +@MODULE_WINREG_TRUE@winreg ../PC/winreg.c @MODULE_MSVCRT_TRUE@msvcrt -DPy_BUILD_CORE ../PC/msvcrtmodule.c @MODULE__WINAPI_TRUE@_winapi _winapi.c diff --git a/PC/winreg.c b/PC/winreg.c index 33ec8ffa07dc99..e8f36c19614e0b 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -19,25 +19,6 @@ #include "structmember.h" // PyMemberDef #include -#ifndef SIZEOF_HKEY -/* used only here */ -#if defined(MS_WIN64) -# define SIZEOF_HKEY 8 -#elif defined(MS_WIN32) -# define SIZEOF_HKEY 4 -#else -# error "SIZEOF_HKEY is not defined" -#endif -#endif - -#ifndef REG_LEGAL_CHANGE_FILTER -#define REG_LEGAL_CHANGE_FILTER (\ - REG_NOTIFY_CHANGE_NAME |\ - REG_NOTIFY_CHANGE_ATTRIBUTES |\ - REG_NOTIFY_CHANGE_LAST_SET |\ - REG_NOTIFY_CHANGE_SECURITY ) -#endif - #if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) typedef struct { diff --git a/configure.ac b/configure.ac index 10a6e8464f9add..5dcec81ce13d23 100644 --- a/configure.ac +++ b/configure.ac @@ -4633,13 +4633,6 @@ case $host in CFLAGS_NODIST="$CFLAGS_NODIST -D_WIN32_WINNT=0x0602";; esac -# Determine if windows modules should be used. -AC_SUBST(USE_WIN32_MODULE) -USE_WIN32_MODULE='#' -case $host in - *-*-mingw*) USE_WIN32_MODULE=;; -esac - # Templates for things AC_DEFINEd more than once. # For a single AC_DEFINE, no template is needed. AH_TEMPLATE([_REENTRANT], @@ -8059,6 +8052,7 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) dnl windows specific modules +PY_STDLIB_MOD([winreg], [test "$MACHDEP" = "win32"]) PY_STDLIB_MOD([msvcrt], [test "$MACHDEP" = "win32"]) PY_STDLIB_MOD([_winapi], [test "$MACHDEP" = "win32"]) PY_STDLIB_MOD([_msi], [test "$MACHDEP" = "win32"], [], [], From e96148690e159d55178dea0412b0e68405a4272b Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 10 Oct 2025 08:52:03 +0200 Subject: [PATCH 127/127] CI: use a newer ubuntu for the cross build 24.10 is EOL --- .github/workflows/mingw.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index b0bf5df4dd61dc..69c034e05f54a6 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -166,7 +166,7 @@ jobs: cross-gcc-x86_64: runs-on: ubuntu-latest container: - image: ubuntu:24.10 + image: ubuntu:25.04 steps: - uses: actions/checkout@v4 - name: Install deps