Update 3.11 patch with recent changes (#261)

* Adds recent iOS patches from 3.14 development
* Adds CI workflow changes from main branch
* Adds modulemap generation
* Adds new-style platform-site handling

Author: freakboy3742

.github/workflows/ci.yaml

@@ -1,10 +1,6 @@
 name: CI
 on:
   pull_request:
-  push:
-    branches:
-      - main
-      - 3.*
   workflow_call:
     inputs:
       build-number:
@@ -109,19 +105,22 @@ jobs:
     - uses: actions/[email protected]
 
     - name: Set up Python
-      uses: actions/setup-python@v5.3.0
+      uses: actions/setup-python@v5.4.0
       with:
         # Appending -dev ensures that we can always build the dev release.
         # It's a no-op for versions that have been published.
         python-version: ${{ needs.config.outputs.PYTHON_VER }}-dev
+        # Ensure that we *always* use the latest build, not a cached version.
+        # It's an edge case, but when a new alpha is released, we need to use it ASAP.
+        check-latest: true
 
     - name: Build ${{ matrix.target }}
       run: |
         # Do the build for the requested target.
         make ${{ matrix.target }} BUILD_NUMBER=${{ needs.config.outputs.BUILD_NUMBER }}
 
     - name: Upload build artefacts
-      uses: actions/upload-artifact@v4.4.3
+      uses: actions/upload-artifact@v4.6.1
       with:
         name: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
         path: dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz

.github/workflows/publish.yaml

@@ -11,7 +11,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python environment
-      uses: actions/setup-python@v5.3.0
+      uses: actions/setup-python@v5.4.0
       with:
         python-version: "3.X"
 

.github/workflows/release.yaml

@@ -40,14 +40,14 @@ jobs:
     needs: [ config, ci ]
     steps:
     - name: Get build artifacts
-      uses: actions/[email protected].8
+      uses: actions/[email protected].9
       with:
         pattern: Python-*
         path: dist
         merge-multiple: true
 
     - name: Create Release
-      uses: ncipollo/release-action@v1.14.0
+      uses: ncipollo/release-action@v1.16.0
       with:
         name: ${{ needs.ci.outputs.PYTHON_VER }}-${{ needs.config.outputs.BUILD_NUMBER }}
         tag: ${{ needs.ci.outputs.PYTHON_VER }}-${{ needs.config.outputs.BUILD_NUMBER }}

Makefile

@@ -27,10 +27,10 @@ PYTHON_VER=$(basename $(PYTHON_VERSION))
 # The binary releases of dependencies, published at:
 # https://github.com/beeware/cpython-apple-source-deps/releases
 BZIP2_VERSION=1.0.8-1
-LIBFFI_VERSION=3.4.6-1
+LIBFFI_VERSION=3.4.7-1
 MPDECIMAL_VERSION=4.0.0-1
-OPENSSL_VERSION=3.0.15-1
-XZ_VERSION=5.6.2-1
+OPENSSL_VERSION=3.0.16-1
+XZ_VERSION=5.6.4-1
 
 # Supported OS
 OS_LIST=macOS iOS tvOS watchOS
@@ -40,24 +40,18 @@ CURL_FLAGS=--disable --fail --location --create-dirs --progress-bar
 # macOS targets
 TARGETS-macOS=macosx.x86_64 macosx.arm64
 VERSION_MIN-macOS=11.0
-CFLAGS-macOS=-mmacosx-version-min=$(VERSION_MIN-macOS)
 
 # iOS targets
 TARGETS-iOS=iphonesimulator.x86_64 iphonesimulator.arm64 iphoneos.arm64
 VERSION_MIN-iOS=13.0
-CFLAGS-iOS=-mios-version-min=$(VERSION_MIN-iOS)
 
 # tvOS targets
 TARGETS-tvOS=appletvsimulator.x86_64 appletvsimulator.arm64 appletvos.arm64
 VERSION_MIN-tvOS=12.0
-CFLAGS-tvOS=-mtvos-version-min=$(VERSION_MIN-tvOS)
-PYTHON_CONFIGURE-tvOS=ac_cv_func_sigaltstack=no
 
 # watchOS targets
 TARGETS-watchOS=watchsimulator.x86_64 watchsimulator.arm64 watchos.arm64_32
 VERSION_MIN-watchOS=4.0
-CFLAGS-watchOS=-mwatchos-version-min=$(VERSION_MIN-watchOS)
-PYTHON_CONFIGURE-watchOS=ac_cv_func_sigaltstack=no
 
 # The architecture of the machine doing the build
 HOST_ARCH=$(shell uname -m)
@@ -72,7 +66,7 @@ PATH=/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin
 all: $(OS_LIST)
 
 .PHONY: \
-	all clean distclean update-patch vars \
+	all clean distclean update-patch vars config \
 	$(foreach os,$(OS_LIST),$(os) clean-$(os) dev-clean-$(os) vars-$(os)) \
 	$(foreach os,$(OS_LIST),$(foreach sdk,$$(sort $$(basename $$(TARGETS-$(os)))),$(sdk) vars-$(sdk)))
 	$(foreach os,$(OS_LIST),$(foreach target,$$(TARGETS-$(os)),$(target) vars-$(target)))
@@ -135,10 +129,10 @@ ARCH-$(target)=$$(subst .,,$$(suffix $(target)))
 ifneq ($(os),macOS)
 	ifeq ($$(findstring simulator,$$(SDK-$(target))),)
 TARGET_TRIPLE-$(target)=$$(ARCH-$(target))-apple-$$(OS_LOWER-$(target))$$(VERSION_MIN-$(os))
-IS_SIMULATOR-$(target)="False"
+IS_SIMULATOR-$(target)=False
 	else
 TARGET_TRIPLE-$(target)=$$(ARCH-$(target))-apple-$$(OS_LOWER-$(target))$$(VERSION_MIN-$(os))-simulator
-IS_SIMULATOR-$(target)="True"
+IS_SIMULATOR-$(target)=True
 	endif
 endif
 
@@ -267,6 +261,9 @@ PYTHON_LIB-$(target)=$$(PYTHON_FRAMEWORK-$(target))/Python
 PYTHON_BIN-$(target)=$$(PYTHON_INSTALL-$(target))/bin
 PYTHON_INCLUDE-$(target)=$$(PYTHON_FRAMEWORK-$(target))/Headers
 PYTHON_STDLIB-$(target)=$$(PYTHON_INSTALL-$(target))/lib/python$(PYTHON_VER)
+PYTHON_PLATFORM_CONFIG-$(target)=$$(PYTHON_INSTALL-$(target))/platform-config/$$(ARCH-$(target))-$$(SDK-$(target))
+PYTHON_PLATFORM_SITECUSTOMIZE-$(target)=$$(PYTHON_PLATFORM_CONFIG-$(target))/sitecustomize.py
+
 
 $$(PYTHON_SRCDIR-$(target))/configure: \
 		downloads/Python-$(PYTHON_VERSION).tar.gz \
@@ -325,23 +322,35 @@ $$(PYTHON_LIB-$(target)): $$(PYTHON_SRCDIR-$(target))/python.exe
 		# Remove any .orig files produced by the compliance patching process
 		find $$(PYTHON_INSTALL-$(target)) -name "*.orig" -exec rm {} \;
 
-endif
-
-PYTHON_SITECUSTOMIZE-$(target)=$(PROJECT_DIR)/support/$(PYTHON_VER)/$(os)/platform-site/$(target)/sitecustomize.py
 
-$$(PYTHON_SITECUSTOMIZE-$(target)):
-	@echo ">>> Create cross-platform sitecustomize.py for $(target)"
-	mkdir -p $$(dir $$(PYTHON_SITECUSTOMIZE-$(target)))
-	cat $(PROJECT_DIR)/patch/Python/sitecustomize.$(os).py \
+$$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target)):
+	@echo ">>> Create cross-plaform config for $(target)"
+	mkdir -p $$(PYTHON_PLATFORM_CONFIG-$(target))
+	# Create the cross-platform site definition
+	echo "import _cross_$$(ARCH-$(target))_$$(SDK-$(target)); import _cross_venv;" \
+		> $$(PYTHON_PLATFORM_CONFIG-$(target))/_cross_venv.pth
+	cp $(PROJECT_DIR)/patch/Python/make_cross_venv.py \
+		$$(PYTHON_PLATFORM_CONFIG-$(target))/make_cross_venv.py
+	cp $(PROJECT_DIR)/patch/Python/_cross_venv.py \
+		$$(PYTHON_PLATFORM_CONFIG-$(target))/_cross_venv.py
+	cp $$(PYTHON_STDLIB-$(target))/_sysconfig* \
+		$$(PYTHON_PLATFORM_CONFIG-$(target))
+	cat $(PROJECT_DIR)/patch/Python/_cross_target.py.tmpl \
 		| sed -e "s/{{os}}/$(os)/g" \
+		| sed -e "s/{{platform}}/$$(OS_LOWER-$(target))/g" \
 		| sed -e "s/{{arch}}/$$(ARCH-$(target))/g" \
+		| sed -e "s/{{sdk}}/$$(SDK-$(target))/g" \
 		| sed -e "s/{{version_min}}/$$(VERSION_MIN-$(os))/g" \
 		| sed -e "s/{{is_simulator}}/$$(IS_SIMULATOR-$(target))/g" \
-		| sed -e "s/{{multiarch}}/$$(ARCH-$(target))-$$(SDK-$(target))/g" \
-		| sed -e "s/{{tag}}/$$(OS_LOWER-$(target))-$$(VERSION_MIN-$(os))-$$(ARCH-$(target))-$$(SDK-$(target))/g" \
-		> $$(PYTHON_SITECUSTOMIZE-$(target))
+		> $$(PYTHON_PLATFORM_CONFIG-$(target))/_cross_$$(ARCH-$(target))_$$(SDK-$(target)).py
+	cat $(PROJECT_DIR)/patch/Python/sitecustomize.py.tmpl \
+		| sed -e "s/{{arch}}/$$(ARCH-$(target))/g" \
+		| sed -e "s/{{sdk}}/$$(SDK-$(target))/g" \
+		> $$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target))
+
+endif
 
-$(target): $$(PYTHON_SITECUSTOMIZE-$(target)) $$(PYTHON_LIB-$(target))
+$(target): $$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target)) $$(PYTHON_LIB-$(target))
 
 ###########################################################################
 # Target: Debug
@@ -370,6 +379,8 @@ vars-$(target):
 	@echo "PYTHON_BIN-$(target): $$(PYTHON_BIN-$(target))"
 	@echo "PYTHON_INCLUDE-$(target): $$(PYTHON_INCLUDE-$(target))"
 	@echo "PYTHON_STDLIB-$(target): $$(PYTHON_STDLIB-$(target))"
+	@echo "PYTHON_PLATFORM_CONFIG-$(target): $$(PYTHON_PLATFORM_CONFIG-$(target))"
+	@echo "PYTHON_PLATFORM_SITECUSTOMIZE-$(target): $$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target))"
 	@echo
 
 endef # build-target
@@ -415,6 +426,7 @@ PYTHON_FRAMEWORK-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/Python.framework
 PYTHON_INSTALL_VERSION-$(sdk)=$$(PYTHON_FRAMEWORK-$(sdk))/Versions/$(PYTHON_VER)
 PYTHON_LIB-$(sdk)=$$(PYTHON_INSTALL_VERSION-$(sdk))/Python
 PYTHON_INCLUDE-$(sdk)=$$(PYTHON_INSTALL_VERSION-$(sdk))/include/python$(PYTHON_VER)
+PYTHON_MODULEMAP-$(sdk)=$$(PYTHON_INCLUDE-$(sdk))/module.modulemap
 PYTHON_STDLIB-$(sdk)=$$(PYTHON_INSTALL_VERSION-$(sdk))/lib/python$(PYTHON_VER)
 
 else
@@ -425,11 +437,13 @@ else
 # The non-macOS frameworks don't use the versioning structure.
 
 PYTHON_INSTALL-$(sdk)=$(PROJECT_DIR)/install/$(os)/$(sdk)/python-$(PYTHON_VERSION)
+PYTHON_MODULEMAP-$(sdk)=$$(PYTHON_INCLUDE-$(sdk))/module.modulemap
 PYTHON_FRAMEWORK-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/Python.framework
 PYTHON_LIB-$(sdk)=$$(PYTHON_FRAMEWORK-$(sdk))/Python
 PYTHON_BIN-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/bin
 PYTHON_INCLUDE-$(sdk)=$$(PYTHON_FRAMEWORK-$(sdk))/Headers
 PYTHON_STDLIB-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/lib/python$(PYTHON_VER)
+PYTHON_PLATFORM_CONFIG-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/platform-config
 
 $$(PYTHON_LIB-$(sdk)): $$(foreach target,$$(SDK_TARGETS-$(sdk)),$$(PYTHON_LIB-$$(target)))
 	@echo ">>> Build Python fat library for the $(sdk) SDK"
@@ -454,6 +468,15 @@ $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h: $$(PYTHON_LIB-$(sdk))
 	# Copy headers as-is from the first target in the $(sdk) SDK
 	cp -r $$(PYTHON_INCLUDE-$$(firstword $$(SDK_TARGETS-$(sdk)))) $$(PYTHON_INCLUDE-$(sdk))
 
+	# Create the modulemap file
+	cp -r patch/Python/module.modulemap.prefix $$(PYTHON_MODULEMAP-$(sdk))
+	echo "" >> $$(PYTHON_MODULEMAP-$(sdk))
+	cd $$(PYTHON_SRCDIR-$$(firstword $$(SDK_TARGETS-$(sdk))))/Include && \
+		find cpython -name "*.h"  | sort | sed -e 's/^/    exclude header "/' | sed 's/$$$$/"/' >> $$(PYTHON_MODULEMAP-$(sdk)) && \
+		echo "" >> $$(PYTHON_MODULEMAP-$(sdk)) && \
+		find internal -name "*.h"  | sort | sed -e 's/^/    exclude header "/' | sed 's/$$$$/"/' >> $$(PYTHON_MODULEMAP-$(sdk))
+	echo "\n}" >> $$(PYTHON_MODULEMAP-$(sdk))
+
 	# Link the PYTHONHOME version of the headers
 	mkdir -p $$(PYTHON_INSTALL-$(sdk))/include
 	ln -si ../Python.framework/Headers $$(PYTHON_INSTALL-$(sdk))/include/python$(PYTHON_VER)
@@ -465,7 +488,7 @@ $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h: $$(PYTHON_LIB-$(sdk))
 	cp $$(PYTHON_SRCDIR-$$(firstword $$(SDK_TARGETS-$(sdk))))/$(os)/Resources/pyconfig.h $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h
 
 
-$$(PYTHON_STDLIB-$(sdk))/LICENSE.TXT: $$(PYTHON_LIB-$(sdk)) $$(PYTHON_FRAMEWORK-$(sdk))/Info.plist $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h
+$$(PYTHON_STDLIB-$(sdk))/LICENSE.TXT: $$(PYTHON_LIB-$(sdk)) $$(PYTHON_FRAMEWORK-$(sdk))/Info.plist $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h $$(foreach target,$$(SDK_TARGETS-$(sdk)),$$(PYTHON_PLATFORM_SITECUSTOMIZE-$$(target)))
 	@echo ">>> Build Python stdlib for the $(sdk) SDK"
 	mkdir -p $$(PYTHON_STDLIB-$(sdk))/lib-dynload
 	# Copy stdlib from the first target associated with the $(sdk) SDK
@@ -480,6 +503,10 @@ $$(PYTHON_STDLIB-$(sdk))/LICENSE.TXT: $$(PYTHON_LIB-$(sdk)) $$(PYTHON_FRAMEWORK-
 	# Copy the individual _sysconfigdata modules into names that include the architecture
 	$$(foreach target,$$(SDK_TARGETS-$(sdk)),cp $$(PYTHON_STDLIB-$$(target))/_sysconfigdata_* $$(PYTHON_STDLIB-$(sdk))/; )
 
+	# Copy the platform site folders for each architecture
+	mkdir -p $$(PYTHON_PLATFORM_CONFIG-$(sdk))
+	$$(foreach target,$$(SDK_TARGETS-$(sdk)),cp -r $$(PYTHON_PLATFORM_CONFIG-$$(target)) $$(PYTHON_PLATFORM_CONFIG-$(sdk)); )
+
 	# Merge the binary modules from each target in the $(sdk) SDK into a single binary
 	$$(foreach module,$$(wildcard $$(PYTHON_STDLIB-$$(firstword $$(SDK_TARGETS-$(sdk))))/lib-dynload/*),lipo -create -output $$(PYTHON_STDLIB-$(sdk))/lib-dynload/$$(notdir $$(module)) $$(foreach target,$$(SDK_TARGETS-$(sdk)),$$(PYTHON_STDLIB-$$(target))/lib-dynload/$$(notdir $$(module))); )
 
@@ -564,6 +591,15 @@ $$(PYTHON_XCFRAMEWORK-$(os))/Info.plist: \
 	# Rewrite the framework to make it standalone
 	patch/make-relocatable.sh $$(PYTHON_INSTALL_VERSION-macosx) 2>&1 > /dev/null
 
+	# Create the modulemap file
+	cp -r patch/Python/module.modulemap.prefix $$(PYTHON_MODULEMAP-macosx)
+	echo "" >> $$(PYTHON_MODULEMAP-macosx)
+	cd $$(PYTHON_INCLUDE-macosx) && \
+		find cpython -name "*.h"  | sort | sed -e 's/^/    exclude header "/' | sed 's/$$$$/"/' >> $$(PYTHON_MODULEMAP-macosx) && \
+		echo "" >> $$(PYTHON_MODULEMAP-macosx) && \
+		find internal -name "*.h"  | sort | sed -e 's/^/    exclude header "/' | sed 's/$$$$/"/' >> $$(PYTHON_MODULEMAP-macosx)
+	echo "\n}" >> $$(PYTHON_MODULEMAP-macosx)
+
 	# Re-apply the signature on the binaries.
 	codesign -s - --preserve-metadata=identifier,entitlements,flags,runtime -f $$(PYTHON_LIB-macosx) \
 		2>&1 | tee $$(PYTHON_INSTALL-macosx)/python-$(os).codesign.log
@@ -587,7 +623,7 @@ support/$(PYTHON_VER)/macOS/VERSIONS:
 dist/Python-$(PYTHON_VER)-macOS-support.$(BUILD_NUMBER).tar.gz: \
 	$$(PYTHON_XCFRAMEWORK-macOS)/Info.plist \
 	support/$(PYTHON_VER)/macOS/VERSIONS \
-	$$(foreach target,$$(TARGETS-macOS), $$(PYTHON_SITECUSTOMIZE-$$(target)))
+	$$(foreach target,$$(TARGETS-macOS), $$(PYTHON_PLATFORM_SITECUSTOMIZE-$$(target)))
 
 	@echo ">>> Create final distribution artefact for macOS"
 	mkdir -p dist
@@ -610,9 +646,7 @@ $$(PYTHON_XCFRAMEWORK-$(os))/Info.plist: \
 	$$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/include $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); )
 	$$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/bin $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); )
 	$$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/lib $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); )
-
-	@echo ">>> Create helper links in XCframework for $(os)"
-	$$(foreach sdk,$$(SDKS-$(os)),ln -si $$(SDK_SLICE-$$(sdk)) $$(PYTHON_XCFRAMEWORK-$(os))/$$(sdk); )
+	$$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/platform-config $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); )
 
 ifeq ($(os),iOS)
 	@echo ">>> Clone testbed project for $(os)"
@@ -632,7 +666,7 @@ endif
 
 dist/Python-$(PYTHON_VER)-$(os)-support.$(BUILD_NUMBER).tar.gz: \
 	$$(PYTHON_XCFRAMEWORK-$(os))/Info.plist \
-	$$(foreach target,$$(TARGETS-$(os)), $$(PYTHON_SITECUSTOMIZE-$$(target)))
+	$$(foreach target,$$(TARGETS-$(os)), $$(PYTHON_PLATFORM_SITECUSTOMIZE-$$(target)))
 
 	@echo ">>> Create final distribution artefact for $(os)"
 	mkdir -p dist

README.rst

@@ -83,15 +83,6 @@ Each support package contains:
 
 * ``VERSIONS``, a text file describing the specific versions of code used to build the
   support package;
-* ``platform-site``, a folder that contains site customization scripts that can be used
-  to make your local Python install look like it is an on-device install for each of the
-  underlying target architectures supported by the platform. This is needed because when
-  you run ``pip`` you'll be on a macOS machine with a specific architecture; if ``pip``
-  tries to install a binary package, it will install a macOS binary wheel (which won't
-  work on iOS/tvOS/watchOS). However, if you add the ``platform-site`` folder to your
-  ``PYTHONPATH`` when invoking pip, the site customization will make your Python install
-  return ``platform`` and ``sysconfig`` responses consistent with on-device behavior,
-  which will cause ``pip`` to install platform-appropriate packages.
 * ``Python.xcframework``, a multi-architecture build of the Python runtime library
 
 On iOS/tvOS/watchOS, the ``Python.xcframework`` contains a
@@ -105,6 +96,32 @@ needed to build packages. This is required because Xcode uses the ``xcrun``
 alias to dynamically generate the name of binaries, but a lot of C tooling
 expects that ``CC`` will not contain spaces.
 
+Each slice of an iOS/tvOS/watchOS XCframework also contains a
+``platform-config`` folder with a subfolder for each supported architecture in
+that slice. These subfolders can be used to make a macOS Python environment
+behave as if it were on an iOS/tvOS/watchOS device. This works in one of two
+ways:
+
+1. **A sitecustomize.py script**. If the ``platform-config`` subfolder is on
+   your ``PYTHONPATH`` when a Python interpreter is started, a site
+   customization will be applied that patches methods in ``sys``, ``sysconfig``
+   and ``platform`` that are used to identify the system.
+
+2. **A make_cross_venv.py script**. If you call ``make_cross_venv.py``,
+   providing the location of a virtual environment, the script will add some
+   files to the ``site-packages`` folder of that environment that will
+   automatically apply the same set of patches as the ``sitecustomize.py``
+   script whenever the environment is activated, without any need to modify
+   ``PYTHONPATH``. If you use ``build`` to create an isolated PEP 517
+   environment to build a wheel, these patches will also be applied to the
+   isolated build environment that is created.
+
+iOS distributions also contain a copy of the iOS ``testbed`` project - an Xcode
+project that can be used to run test suites of Python code. See the `CPython
+documentation on testing packages
+<https://docs.python.org/3/using/ios.html#testing-a-python-package>`__ for
+details on how to use this testbed.
+
 For a detailed instructions on using the support package in your own project,
 see the `usage guide <./USAGE.md>`__