Merge branch 'main' into 3.9

Includes updates to:
* Python 3.9.18
* XZ 5.4.4
* OpenSSL 3.1.2
* LibFFI 3.4.4

Author: freakboy3742

Makefile

@@ -39,23 +39,23 @@ BUILD_NUMBER=custom
 # PYTHON_VERSION is the full version number (e.g., 3.10.0b3)
 # PYTHON_MICRO_VERSION is the full version number, without any alpha/beta/rc suffix. (e.g., 3.10.0)
 # PYTHON_VER is the major/minor version (e.g., 3.10)
-PYTHON_VERSION=3.9.14
+PYTHON_VERSION=3.9.18
 PYTHON_MICRO_VERSION=$(shell echo $(PYTHON_VERSION) | grep -Eo "\d+\.\d+\.\d+")
 PYTHON_VER=$(basename $(PYTHON_VERSION))
 
 BZIP2_VERSION=1.0.8
 
-XZ_VERSION=5.2.6
+XZ_VERSION=5.4.4
 
 # Preference is to use OpenSSL 3; however, Cryptography 3.4.8 (and
 # probably some other packages as well) only works with 1.1.1, so
 # we need to preserve the ability to build the older OpenSSL (for now...)
-OPENSSL_VERSION=3.0.5
+OPENSSL_VERSION=3.1.2
 # OPENSSL_VERSION_NUMBER=1.1.1
-# OPENSSL_REVISION=q
+# OPENSSL_REVISION=v
 # OPENSSL_VERSION=$(OPENSSL_VERSION_NUMBER)$(OPENSSL_REVISION)
 
-LIBFFI_VERSION=3.4.2
+LIBFFI_VERSION=3.4.4
 
 # Supported OS and dependencies
 DEPENDENCIES=BZip2 XZ OpenSSL libFFI
@@ -107,7 +107,7 @@ all: $(OS_LIST)
 
 # Clean all builds
 clean:
-	rm -rf build install merge dist support
+	rm -rf build install merge dist support wheels
 
 # Full clean - includes all downloaded products
 distclean: clean
@@ -314,6 +314,7 @@ $$(XZ_SRCDIR-$(target))/Makefile: downloads/xz-$(XZ_VERSION).tar.gz
 			--disable-shared \
 			--enable-static \
 			--host=$$(TARGET_TRIPLE-$(target)) \
+			--build=$(HOST_ARCH)-apple-darwin \
 			--prefix="$$(XZ_INSTALL-$(target))" \
 			2>&1 | tee -a ../xz-$(XZ_VERSION).config.log
 
@@ -910,8 +911,8 @@ os=$1
 SDKS-$(os)=$$(sort $$(basename $$(TARGETS-$(os))))
 
 # Predeclare the Python XCFramework files so they can be referenced in SDK targets
-PYTHON_XCFRAMEWORK-$(os)=support/$(os)/Python.xcframework
-PYTHON_STDLIB-$(os)=support/$(os)/python-stdlib
+PYTHON_XCFRAMEWORK-$(os)=support/$(PYTHON_VER)/$(os)/Python.xcframework
+PYTHON_STDLIB-$(os)=support/$(PYTHON_VER)/$(os)/python-stdlib
 
 # Expand the build-sdk macro for all the sdks on this OS (e.g., iphoneos, iphonesimulator)
 $$(foreach sdk,$$(SDKS-$(os)),$$(eval $$(call build-sdk,$$(sdk),$(os))))
@@ -1033,7 +1034,7 @@ $$(PYTHON_XCFRAMEWORK-$(os)): \
 	mkdir -p $$(dir $$(PYTHON_XCFRAMEWORK-$(os)))
 	xcodebuild -create-xcframework \
 		-output $$@ $$(foreach sdk,$$(SDKS-$(os)),-library $$(PYTHON_FATLIB-$$(sdk)) -headers $$(PYTHON_FATINCLUDE-$$(sdk))) \
-		2>&1 | tee -a support/python-$(os).xcframework.log
+		2>&1 | tee -a support/$(PYTHON_VER)/python-$(os).xcframework.log
 
 $$(PYTHON_STDLIB-$(os)): \
 		$$(PYTHON_XCFRAMEWORK-$(os)) \
@@ -1059,34 +1060,34 @@ $$(PYTHON_STDLIB-$(os)): \
 
 dist/Python-$(PYTHON_VER)-$(os)-support.$(BUILD_NUMBER).tar.gz: $$(PYTHON_XCFRAMEWORK-$(os)) $$(PYTHON_STDLIB-$(os))
 	@echo ">>> Create VERSIONS file for $(os)"
-	echo "Python version: $(PYTHON_VERSION) " > support/$(os)/VERSIONS
-	echo "Build: $(BUILD_NUMBER)" >> support/$(os)/VERSIONS
-	echo "Min $(os) version: $$(VERSION_MIN-$(os))" >> support/$(os)/VERSIONS
-	echo "---------------------" >> support/$(os)/VERSIONS
+	echo "Python version: $(PYTHON_VERSION) " > support/$(PYTHON_VER)/$(os)/VERSIONS
+	echo "Build: $(BUILD_NUMBER)" >> support/$(PYTHON_VER)/$(os)/VERSIONS
+	echo "Min $(os) version: $$(VERSION_MIN-$(os))" >> support/$(PYTHON_VER)/$(os)/VERSIONS
+	echo "---------------------" >> support/$(PYTHON_VER)/$(os)/VERSIONS
 ifeq ($(os),macOS)
-	echo "libFFI: macOS native" >> support/$(os)/VERSIONS
+	echo "libFFI: macOS native" >> support/$(PYTHON_VER)/$(os)/VERSIONS
 else
-	echo "libFFI: $(LIBFFI_VERSION)" >> support/$(os)/VERSIONS
+	echo "libFFI: $(LIBFFI_VERSION)" >> support/$(PYTHON_VER)/$(os)/VERSIONS
 endif
-	echo "BZip2: $(BZIP2_VERSION)" >> support/$(os)/VERSIONS
-	echo "OpenSSL: $(OPENSSL_VERSION)" >> support/$(os)/VERSIONS
-	echo "XZ: $(XZ_VERSION)" >> support/$(os)/VERSIONS
+	echo "BZip2: $(BZIP2_VERSION)" >> support/$(PYTHON_VER)/$(os)/VERSIONS
+	echo "OpenSSL: $(OPENSSL_VERSION)" >> support/$(PYTHON_VER)/$(os)/VERSIONS
+	echo "XZ: $(XZ_VERSION)" >> support/$(PYTHON_VER)/$(os)/VERSIONS
 
 ifneq ($(os),macOS)
 	@echo ">>> Create cross-platform site sitecustomize.py for $(os)"
-	mkdir -p support/$(os)/platform-site
+	mkdir -p support/$(PYTHON_VER)/$(os)/platform-site
 	cat $(PROJECT_DIR)/patch/Python/sitecustomize.py \
 		| sed -e "s/{{os}}/$(os)/g" \
 		| sed -e "s/{{tag}}/$$(shell echo $(os) | tr '[:upper:]' '[:lower:]')_$$(shell echo $$(VERSION_MIN-$(os)) | sed "s/\./_/g")/g" \
-		> support/$(os)/platform-site/sitecustomize.py
+		> support/$(PYTHON_VER)/$(os)/platform-site/sitecustomize.py
 endif
 
 	@echo ">>> Create final distribution artefact for $(os)"
 	mkdir -p dist
 	# Build a "full" tarball with all content for test purposes
-	tar zcvf dist/Python-$(PYTHON_VER)-$(os)-support.test-$(BUILD_NUMBER).tar.gz -X patch/Python/test.exclude -C support/$(os) `ls -A support/$(os)`
+	tar zcvf dist/Python-$(PYTHON_VER)-$(os)-support.test-$(BUILD_NUMBER).tar.gz -X patch/Python/test.exclude -C support/$(PYTHON_VER)/$(os) `ls -A support/$(PYTHON_VER)/$(os)/`
 	# Build a distributable tarball
-	tar zcvf $$@ -X patch/Python/release.common.exclude -X patch/Python/release.$(os).exclude -C support/$(os) `ls -A support/$(os)`
+	tar zcvf $$@ -X patch/Python/release.common.exclude -X patch/Python/release.$(os).exclude -C support/$(PYTHON_VER)/$(os) `ls -A support/$(PYTHON_VER)/$(os)/`
 
 Python-$(os): dist/Python-$(PYTHON_VER)-$(os)-support.$(BUILD_NUMBER).tar.gz
 
@@ -1099,8 +1100,7 @@ clean-Python-$(os):
 		install/$(os)/*/python-$(PYTHON_VER)*.*.log \
 		merge/$(os)/*/python-$(PYTHON_VER)* \
 		merge/$(os)/*/python-$(PYTHON_VER)*.*.log \
-		support/$(os) \
-		support/*-$(os).*.log \
+		support/$(PYTHON_VER)/$(os) \
 		dist/Python-$(PYTHON_VER)-$(os)-*
 
 dev-clean-Python-$(os):
@@ -1112,17 +1112,15 @@ dev-clean-Python-$(os):
 		install/$(os)/*/python-$(PYTHON_VERSION).*.log \
 		merge/$(os)/*/python-$(PYTHON_VERSION) \
 		merge/$(os)/*/python-$(PYTHON_VERSION).*.log \
-		support/$(os) \
-		support/*-$(os).*.log \
+		support/$(PYTHON_VER)/$(os) \
 		dist/Python-$(PYTHON_VER)-$(os)-*
 
 merge-clean-Python-$(os):
 	@echo ">>> Partially clean Python build products on $(os) so that merge modifications can be made"
 	rm -rf \
 		merge/$(os)/*/python-$(PYTHON_VERSION) \
 		merge/$(os)/*/python-$(PYTHON_VERSION).*.log \
-		support/$(os) \
-		support/*-$(os).*.log \
+		support/$(PYTHON_VER)/$(os) \
 		dist/Python-$(PYTHON_VER)-$(os)-*
 
 ###########################################################################

README.rst

@@ -4,7 +4,7 @@ Python Apple Support
 This is a meta-package for building a version of Python that can be embedded
 into a macOS, iOS, tvOS or watchOS project.
 
-**This branch builds a packaged version of Python 3.9.14**.
+**This branch builds a packaged version of Python 3.9.18**.
 Other Python versions are available by cloning other branches of the main
 repository:
 
@@ -106,43 +106,17 @@ install return ``platform`` and ``sysconfig`` responses consistent with
 on-device behavior, which will cause ``pip`` to install platform-appropriate
 packages.
 
-To add a support package to your own Xcode project:
-
-1. Drag ``Python.xcframework`` and ``python-stdlib`` into your Xcode project
-   tree.
-2. Ensure that these two objects are added to any targets that need to use
-   them;
-3. Add a custom build phase to purge any binary modules for the platform you are
-   *not* targetting; and
-4. Add a custom build phase to sign any of the binary modules in your app.
-5. Add CPython API code to your app to create an instance of the Python
-   interpreter.
-
-For examples of the scripts needed for steps 3 and 4, and the code needed for
-step 5, compare with a project generated with Briefcase.
-
-On macOS, you must also either:
-1. Enable the "Disable Library Validation" entitlement (found on the "Signing
-   & Capabilities" tab in XCode); or
-2. Sign your app with a Development or Distribution certificate. This will
-   require a paid Apple Developer subscription.
-
-It is not possible to use an ad-hoc signing certificate with the "Disable
-Library Validation" entitlement disabled.
-
-On iOS/tvOS/watchOS, you can use the default developer certificate for deploying
-to a device simulator. However, to deploy to a physical device (including your
-own), you will require a Development or Distribution certificate, which requires
-a paid Apple Developer subscription.
+For a detailed instructions on using the support package in your own project,
+see the `usage guide <./USAGE.md>`__
 
 Building binary wheels
 ----------------------
 
 When building binary wheels, you may need to use the libraries built by this
 project as inputs (e.g., the `cffi` module uses `libffi`). To support this, this
 project is able to package these dependencies as "wheels" that can be added to
-the `server/pypi/dist` directory of the [binary dependency builder
-project](https://github.com/freakboy3742/chaquopy).
+the `server/pypi/dist` directory of the `binary dependency builder
+project <https://github.com/freakboy3742/chaquopy>`__.
 
 To build these wheels, run:
 

USAGE.md

@@ -0,0 +1,120 @@
+# Usage Guide
+
+## The easy way
+
+The easist way to use these packages is by creating a project with `Briefcase
+<https://github.com/beeware/briefcase>`__. Briefcase will download pre-compiled
+versions of these support packages, and add them to an Xcode project (or
+pre-build stub application, in the case of macOS).
+
+## The manual way
+
+The Python support package *can* be manually added to any Xcode project;
+however, you'll need to perform some steps manually (essentially reproducing what
+Briefcase is doing)
+
+**NOTE** Briefcase usage is the officially supported approach for using this
+support package. If you are experiencing diffculties, one approach for debugging
+is to generate a "Hello World" project with Briefcase, and compare the project that
+Briefcase has generated with your own project.
+
+To add this support package to your own project:
+
+1. [Download a release tarball for your desired Python version and Apple platform](https://github.com/beeware/Python-Apple-support/releases)
+
+2. Add the `python-stdlib` and `Python.xcframework` to your Xcode project. Both
+   the `python-stdlib` folder and the `Python.xcframework` should be members of
+   any target that needs to use Python.
+
+3. In Xcode, select the root node of the project tree, and select the target you
+   want to build.
+
+4. Select "General" -> "Frameworks, Libraries and Embedded Content", and ensure
+   that `Python.xcframework` is on the list of frameworks. It should be marked
+   "Do not embed".
+
+5. Select "General" -> "Build Phases", and ensure that the `python-stdlib` folder
+   is listed in the "Copy Bundle Resources" step.
+
+6. Add a new "Run script" build phase named "Sign Python Binary Modules", with the following content:
+
+```bash
+set -e
+echo "Signing as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)"
+find "$CODESIGNING_FOLDER_PATH/Contents/Resources/python-stdlib/lib-dynload" -name "*.so" -exec /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der {} \;
+```
+
+You will now be able to access the Python runtime in your Python code.
+
+If you are on iOS, you will be able to deploy to an iOS simulator without specifying
+development team; however, you will need to specify a valid development team to sign
+the binaries for deployment onto a physical device (or for submission to the App Store).
+
+If you are on macOS, you will need to specify a valid development team to run
+the app. If you don't want to specify a development team in your project, you
+will also need to enable the "Disable Library Validation" entitlement under
+"Signing & Capabilities" -> "Hardened Runtime" for your project.
+
+## Accessing the Python runtime
+
+There are 2 ways to access the Python runtime in your project code.
+
+### Embedded C API.
+
+You can use the [Python Embedded C
+API](https://docs.python.org/3/extending/embedding.html) to instantiate a Python
+interpreter. This is the approach taken by Briefcase; you may find the bootstrap
+mainline code generated by Briefcase a helpful guide to what is needed to start
+an interpreter and run Python code.
+
+### PythonKit
+
+An alternate approach is to use
+[PythonKit](https://github.com/pvieito/PythonKit). PythonKit is a package that
+provides a Swift API to running Python code.
+
+To use PythonKit in your project:
+
+1. Add PythonKit to your project using the Swift Package manager. See the
+   PythonKit documentation for details.
+
+2. Create a file called `module.modulemap` inside
+   `Python.xcframework/macos-arm64_x86_64/Headers/`, containing the following
+   code:
+```
+module Python {
+    umbrella header "Python.h"
+    export *
+    link "Python"
+}
+```
+
+3. In your Swift code, initialize the Python runtime. This should generally be
+   done as early as possible in the application's lifecycle, but definitely
+   needs to be done before you invoke Python code:
+```swift
+import Python
+
+guard let stdLibPath = Bundle.main.path(forResource: "python-stdlib", ofType: nil) else { return }
+guard let libDynloadPath = Bundle.main.path(forResource: "python-stdlib/lib-dynload", ofType: nil) else { return }
+setenv("PYTHONHOME", stdLibPath, 1)
+setenv("PYTHONPATH", "\(stdLibPath):\(libDynloadPath)", 1)
+Py_Initialize()
+// we now have a Python interpreter ready to be used
+```
+
+5. Invoke Python code in your app. For example:
+```swift
+import PythonKit
+
+let sys = Python.import("sys")
+print("Python Version: \(sys.version_info.major).\(sys.version_info.minor)")
+print("Python Encoding: \(sys.getdefaultencoding().upper())")
+print("Python Path: \(sys.path)")
+
+_ = Python.import("math") // verifies `lib-dynload` is found and signed successfully
+```
+
+To integrate 3rd party python code and dependencies, you will need to make sure
+`PYTHONPATH` contains their paths; once this has been done, you can run
+`Python.import("<lib name>")`. to import that module from inside swift.