From 659cb8592b74410df017cb48030537d12a943a19 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 19 Sep 2025 13:23:38 +0100 Subject: [PATCH] gh-138171: Migrate iOS testbed location and add Apple build script (#138176) Adds tooling to generate and test an iOS XCframework, in a way that will also facilitate adding other XCframework targets for other Apple platforms (tvOS, watchOS, visionOS and even macOS, potentially). --------- Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> (cherry picked from commit 35c7e52b3ea3fdeb8eb77d2d8c803467a2ba6311) --- .github/CODEOWNERS | 3 + .gitignore | 18 +- Apple/__main__.py | 990 ++++++++++++++++++ Apple/iOS/README.md | 328 ++++++ {iOS => Apple/iOS}/Resources/Info.plist.in | 0 Apple/iOS/Resources/bin/arm64-apple-ios-ar | 2 + Apple/iOS/Resources/bin/arm64-apple-ios-clang | 2 + .../iOS/Resources/bin/arm64-apple-ios-clang++ | 2 + Apple/iOS/Resources/bin/arm64-apple-ios-cpp | 2 + .../bin/arm64-apple-ios-simulator-ar | 2 + .../bin/arm64-apple-ios-simulator-clang | 2 + .../bin/arm64-apple-ios-simulator-clang++ | 2 + .../bin/arm64-apple-ios-simulator-cpp | 2 + .../bin/arm64-apple-ios-simulator-strip | 2 + Apple/iOS/Resources/bin/arm64-apple-ios-strip | 2 + .../bin/x86_64-apple-ios-simulator-ar | 2 + .../bin/x86_64-apple-ios-simulator-clang | 2 + .../bin/x86_64-apple-ios-simulator-clang++ | 2 + .../bin/x86_64-apple-ios-simulator-cpp | 2 + .../bin/x86_64-apple-ios-simulator-strip | 2 + {iOS => Apple/iOS}/Resources/pyconfig.h | 0 .../testbed/Python.xcframework/Info.plist | 0 .../build/iOS-dylib-Info-template.plist | 2 +- .../testbed/Python.xcframework/build/utils.sh | 137 +++ .../Python.xcframework/ios-arm64/README | 0 .../ios-arm64_x86_64-simulator/README | 0 .../testbed/Testbed.lldbinit | 0 .../testbed/TestbedTests/TestbedTests.m | 8 +- {iOS => Apple}/testbed/__main__.py | 195 ++-- .../iOSTestbed.xcodeproj/project.pbxproj | 47 +- .../xcschemes/iOSTestbed.xcscheme | 2 +- {iOS => Apple}/testbed/iOSTestbed.xctestplan | 0 .../testbed/iOSTestbed/AppDelegate.h | 0 .../testbed/iOSTestbed/AppDelegate.m | 0 .../AccentColor.colorset/Contents.json | 0 .../AppIcon.appiconset/Contents.json | 0 .../iOSTestbed/Assets.xcassets/Contents.json | 0 .../Base.lproj/LaunchScreen.storyboard | 0 {iOS => Apple}/testbed/iOSTestbed/app/README | 2 +- .../testbed/iOSTestbed/app_packages/README | 2 +- .../testbed/iOSTestbed/iOSTestbed-Info.plist | 0 {iOS => Apple}/testbed/iOSTestbed/main.m | 0 Doc/using/ios.rst | 151 +-- Makefile.pre.in | 12 +- ...-08-27-11-14-53.gh-issue-138171.Suz8ob.rst | 3 + configure | 8 +- configure.ac | 6 +- iOS/README.rst | 352 ------- iOS/Resources/dylib-Info-template.plist | 26 - 49 files changed, 1710 insertions(+), 612 deletions(-) create mode 100644 Apple/__main__.py create mode 100644 Apple/iOS/README.md rename {iOS => Apple/iOS}/Resources/Info.plist.in (100%) create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-ar create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-clang create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-clang++ create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-cpp create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip create mode 100755 Apple/iOS/Resources/bin/arm64-apple-ios-strip create mode 100755 Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar create mode 100755 Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang create mode 100755 Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ create mode 100755 Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp create mode 100755 Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip rename {iOS => Apple/iOS}/Resources/pyconfig.h (100%) rename {iOS => Apple}/testbed/Python.xcframework/Info.plist (100%) rename iOS/testbed/iOSTestbed/dylib-Info-template.plist => Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist (96%) create mode 100755 Apple/testbed/Python.xcframework/build/utils.sh rename {iOS => Apple}/testbed/Python.xcframework/ios-arm64/README (100%) rename {iOS => Apple}/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README (100%) rename iOS/testbed/iOSTestbed.lldbinit => Apple/testbed/Testbed.lldbinit (100%) rename iOS/testbed/iOSTestbedTests/iOSTestbedTests.m => Apple/testbed/TestbedTests/TestbedTests.m (97%) rename {iOS => Apple}/testbed/__main__.py (59%) rename {iOS => Apple}/testbed/iOSTestbed.xcodeproj/project.pbxproj (79%) rename {iOS => Apple}/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme (97%) rename {iOS => Apple}/testbed/iOSTestbed.xctestplan (100%) rename {iOS => Apple}/testbed/iOSTestbed/AppDelegate.h (100%) rename {iOS => Apple}/testbed/iOSTestbed/AppDelegate.m (100%) rename {iOS => Apple}/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json (100%) rename {iOS => Apple}/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json (100%) rename {iOS => Apple}/testbed/iOSTestbed/Assets.xcassets/Contents.json (100%) rename {iOS => Apple}/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard (100%) rename {iOS => Apple}/testbed/iOSTestbed/app/README (92%) rename {iOS => Apple}/testbed/iOSTestbed/app_packages/README (92%) rename {iOS => Apple}/testbed/iOSTestbed/iOSTestbed-Info.plist (100%) rename {iOS => Apple}/testbed/iOSTestbed/main.m (100%) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-08-27-11-14-53.gh-issue-138171.Suz8ob.rst delete mode 100644 iOS/README.rst delete mode 100644 iOS/Resources/dylib-Info-template.plist diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dc22840162ba5a..b190d8ac1ee955 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -293,6 +293,9 @@ Lib/concurrent/futures/interpreter.py @ericsnowcurrently **/*Android* @mhsmith @freakboy3742 **/*android* @mhsmith @freakboy3742 +# Apple +/Apple @freakboy3742 + # iOS (but not termios) **/iOS* @freakboy3742 **/ios* @freakboy3742 diff --git a/.gitignore b/.gitignore index cdd5c940d038a0..89183d255b0b0e 100644 --- a/.gitignore +++ b/.gitignore @@ -71,15 +71,15 @@ Lib/test/data/* /Makefile /Makefile.pre /iOSTestbed.* -iOS/Frameworks/ -iOS/Resources/Info.plist -iOS/testbed/build -iOS/testbed/Python.xcframework/ios-*/bin -iOS/testbed/Python.xcframework/ios-*/include -iOS/testbed/Python.xcframework/ios-*/lib -iOS/testbed/Python.xcframework/ios-*/Python.framework -iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace -iOS/testbed/iOSTestbed.xcodeproj/xcuserdata +Apple/iOS/Frameworks/ +Apple/iOS/Resources/Info.plist +Apple/testbed/build +Apple/testbed/Python.xcframework/*/bin +Apple/testbed/Python.xcframework/*/include +Apple/testbed/Python.xcframework/*/lib +Apple/testbed/Python.xcframework/*/Python.framework +Apple/testbed/*Testbed.xcodeproj/project.xcworkspace +Apple/testbed/*Testbed.xcodeproj/xcuserdata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile diff --git a/Apple/__main__.py b/Apple/__main__.py new file mode 100644 index 00000000000000..fc19b31be97bb2 --- /dev/null +++ b/Apple/__main__.py @@ -0,0 +1,990 @@ +#!/usr/bin/env python3 +########################################################################## +# Apple XCframework build script +# +# This script simplifies the process of configuring, compiling and packaging an +# XCframework for an Apple platform. +# +# At present, it only supports iOS, but it has been constructed so that it +# could be used on any Apple platform. +# +# The simplest entry point is: +# +# $ python Apple ci iOS +# +# which will: +# * Clean any pre-existing build artefacts +# * Configure and make a Python that can be used for the build +# * Configure and make a Python for each supported iOS architecture and ABI +# * Combine the outputs of the builds from the previous step into a single +# XCframework, merging binaries into a "fat" binary if necessary +# * Clone a copy of the testbed, configured to use the XCframework +# * Construct a tarball containing the release artefacts +# * Run the test suite using the generated XCframework. +# +# This is the complete sequence that would be needed in CI to build and test +# a candidate release artefact. +# +# Each individual step can be invoked individually - there are commands to +# clean, configure-build, make-build, configure-host, make-host, package, and +# test. +# +# There is also a build command that can be used to combine the configure and +# make steps for the build Python, an individual host, all hosts, or all +# builds. +########################################################################## +from __future__ import annotations + +import argparse +import os +import platform +import re +import shlex +import shutil +import signal +import subprocess +import sys +import sysconfig +import time +from collections.abc import Sequence +from contextlib import contextmanager +from datetime import datetime, timezone +from os.path import basename, relpath +from pathlib import Path +from subprocess import CalledProcessError +from typing import Callable + +EnvironmentT = dict[str, str] +ArgsT = Sequence[str | Path] + +SCRIPT_NAME = Path(__file__).name +PYTHON_DIR = Path(__file__).resolve().parent.parent + +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" + +HOSTS: dict[str, dict[str, dict[str, str]]] = { + # Structure of this data: + # * Platform identifier + # * an XCframework slice that must exist for that platform + # * a host triple: the multiarch spec for that host + "iOS": { + "ios-arm64": { + "arm64-apple-ios": "arm64-iphoneos", + }, + "ios-arm64_x86_64-simulator": { + "arm64-apple-ios-simulator": "arm64-iphonesimulator", + "x86_64-apple-ios-simulator": "x86_64-iphonesimulator", + }, + }, +} + + +def subdir(name: str, create: bool = False) -> Path: + """Ensure that a cross-build directory for the given name exists.""" + path = CROSS_BUILD_DIR / name + if not path.exists(): + if not create: + sys.exit( + f"{path} does not exist. Create it by running the appropriate " + f"`configure` subcommand of {SCRIPT_NAME}." + ) + else: + path.mkdir(parents=True) + return path + + +def run( + command: ArgsT, + *, + host: str | None = None, + env: EnvironmentT | None = None, + log: bool | None = True, + **kwargs, +) -> subprocess.CompletedProcess: + """Run a command in an Apple development environment. + + Optionally logs the executed command to the console. + """ + kwargs.setdefault("check", True) + if env is None: + env = os.environ.copy() + + if host: + host_env = apple_env(host) + print_env(host_env) + env.update(host_env) + + if log: + print(">", join_command(command)) + return subprocess.run(command, env=env, **kwargs) + + +def join_command(args: str | Path | ArgsT) -> str: + """Format a command so it can be copied into a shell. + + Similar to `shlex.join`, but also accepts arguments which are Paths, or a + single string/Path outside of a list. + """ + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + +def print_env(env: EnvironmentT) -> None: + """Format the environment so it can be pasted into a shell.""" + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + + +def apple_env(host: str) -> EnvironmentT: + """Construct an Apple development environment for the given host.""" + env = { + "PATH": ":".join( + [ + str(PYTHON_DIR / "Apple/iOS/Resources/bin"), + str(subdir(host) / "prefix"), + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/Library/Apple/usr/bin", + ] + ), + } + + return env + + +def delete_path(name: str) -> None: + """Delete the named cross-build directory, if it exists.""" + path = CROSS_BUILD_DIR / name + if path.exists(): + print(f"Deleting {path} ...") + shutil.rmtree(path) + + +def all_host_triples(platform: str) -> list[str]: + """Return all host triples for the given platform. + + The host triples are the platform definitions used as input to configure + (e.g., "arm64-apple-ios-simulator"). + """ + triples = [] + for slice_name, slice_parts in HOSTS[platform].items(): + triples.extend(list(slice_parts)) + return triples + + +def clean(context: argparse.Namespace, target: str = "all") -> None: + """The implementation of the "clean" command.""" + # If we're explicitly targeting the build, there's no platform or + # distribution artefacts. If we're cleaning tests, we keep all built + # artefacts. Otherwise, the built artefacts must be dirty, so we remove + # them. + if target not in {"build", "test"}: + paths = ["dist", context.platform] + list(HOSTS[context.platform]) + else: + paths = [] + + if target in {"all", "build"}: + paths.append("build") + + if target in {"all", "hosts"}: + paths.extend(all_host_triples(context.platform)) + elif target not in {"build", "test", "package"}: + paths.append(target) + + if target in {"all", "hosts", "test"}: + paths.extend( + [ + path.name + for path in CROSS_BUILD_DIR.glob( + f"{context.platform}-testbed.*" + ) + ] + ) + + for path in paths: + delete_path(path) + + +def build_python_path() -> Path: + """The path to the build Python binary.""" + build_dir = subdir("build") + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {build_dir}" + ) + + return binary + + +@contextmanager +def group(text: str): + """A context manager that outputs a log marker around a section of a build. + + If running in a GitHub Actions environment, the GitHub syntax for + collapsible log sections is used. + """ + if "GITHUB_ACTIONS" in os.environ: + print(f"::group::{text}") + else: + print(f"===== {text} " + "=" * (70 - len(text))) + + yield + + if "GITHUB_ACTIONS" in os.environ: + print("::endgroup::") + else: + print() + + +@contextmanager +def cwd(subdir: Path): + """A context manager that sets the current working directory.""" + orig = os.getcwd() + os.chdir(subdir) + yield + os.chdir(orig) + + +def configure_build_python(context: argparse.Namespace) -> None: + """The implementation of the "configure-build" command.""" + if context.clean: + clean(context, "build") + + with ( + group("Configuring build Python"), + cwd(subdir("build", create=True)), + ): + command = [relpath(PYTHON_DIR / "configure")] + if context.args: + command.extend(context.args) + run(command) + + +def make_build_python(context: argparse.Namespace) -> None: + """The implementation of the "make-build" command.""" + with ( + group("Compiling build Python"), + cwd(subdir("build")), + ): + run(["make", "-j", str(os.cpu_count())]) + + +def apple_target(host: str) -> str: + """Return the Apple platform identifier for a given host triple.""" + for _, platform_slices in HOSTS.items(): + for slice_name, slice_parts in platform_slices.items(): + for host_triple, multiarch in slice_parts.items(): + if host == host_triple: + return ".".join(multiarch.split("-")[::-1]) + + raise KeyError(host) + + +def apple_multiarch(host: str) -> str: + """Return the multiarch descriptor for a given host triple.""" + for _, platform_slices in HOSTS.items(): + for slice_name, slice_parts in platform_slices.items(): + for host_triple, multiarch in slice_parts.items(): + if host == host_triple: + return multiarch + + raise KeyError(host) + + +def unpack_deps( + platform: str, + host: str, + prefix_dir: Path, + cache_dir: Path, +) -> None: + """Unpack binary dependencies into a provided directory. + + Downloads binaries if they aren't already present. Downloads will be stored + in provided cache directory. + + On iOS, as a safety mechanism, any dynamic libraries will be purged from + the unpacked dependencies. + """ + deps_url = "https://github.com/beeware/cpython-apple-source-deps/releases/download" + for name_ver in [ + "BZip2-1.0.8-2", + "libFFI-3.4.7-2", + "OpenSSL-3.0.16-2", + "XZ-5.6.4-2", + "mpdecimal-4.0.0-2", + "zstd-1.5.7-1", + ]: + filename = f"{name_ver.lower()}-{apple_target(host)}.tar.gz" + archive_path = download( + f"{deps_url}/{name_ver}/{filename}", + target_dir=cache_dir, + ) + shutil.unpack_archive(archive_path, prefix_dir) + + # Dynamic libraries will be preferentially linked over static; + # On iOS, ensure that no dylibs are available in the prefix folder. + if platform == "iOS": + for dylib in prefix_dir.glob("**/*.dylib"): + dylib.unlink() + + +def download(url: str, target_dir: Path) -> Path: + """Download the specified URL into the given directory. + + :return: The path to the downloaded archive. + """ + target_path = Path(target_dir).resolve() + target_path.mkdir(exist_ok=True, parents=True) + + out_path = target_path / basename(url) + if not Path(out_path).is_file(): + run( + [ + "curl", + "-Lf", + "--retry", + "5", + "--retry-all-errors", + "-o", + out_path, + url, + ] + ) + else: + print(f"Using cached version of {basename(url)}") + return out_path + + +def configure_host_python( + context: argparse.Namespace, + host: str | None = None, +) -> None: + """The implementation of the "configure-host" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, host) + + host_dir = subdir(host, create=True) + prefix_dir = host_dir / "prefix" + + with group(f"Downloading dependencies ({host})"): + if not prefix_dir.exists(): + prefix_dir.mkdir() + unpack_deps(context.platform, host, prefix_dir, context.cache_dir) + else: + print("Dependencies already installed") + + with ( + group(f"Configuring host Python ({host})"), + cwd(host_dir), + ): + command = [ + # Basic cross-compiling configuration + relpath(PYTHON_DIR / "configure"), + f"--host={host}", + f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", + f"--with-build-python={build_python_path()}", + "--with-system-libmpdec", + "--enable-framework", + # Dependent libraries. + f"--with-openssl={prefix_dir}", + f"LIBLZMA_CFLAGS=-I{prefix_dir}/include", + f"LIBLZMA_LIBS=-L{prefix_dir}/lib -llzma", + f"LIBFFI_CFLAGS=-I{prefix_dir}/include", + f"LIBFFI_LIBS=-L{prefix_dir}/lib -lffi", + f"LIBMPDEC_CFLAGS=-I{prefix_dir}/include", + f"LIBMPDEC_LIBS=-L{prefix_dir}/lib -lmpdec", + f"LIBZSTD_CFLAGS=-I{prefix_dir}/include", + f"LIBZSTD_LIBS=-L{prefix_dir}/lib -lzstd", + ] + + if context.args: + command.extend(context.args) + run(command, host=host) + + +def make_host_python( + context: argparse.Namespace, + host: str | None = None, +) -> None: + """The implementation of the "make-host" command.""" + if host is None: + host = context.host + + with ( + group(f"Compiling host Python ({host})"), + cwd(subdir(host)), + ): + run(["make", "-j", str(os.cpu_count())], host=host) + run(["make", "install"], host=host) + + +def framework_path(host_triple: str, multiarch: str) -> Path: + """The path to a built single-architecture framework product. + + :param host_triple: The host triple (e.g., arm64-apple-ios-simulator) + :param multiarch: The multiarch identifier (e.g., arm64-simulator) + """ + return CROSS_BUILD_DIR / f"{host_triple}/Apple/iOS/Frameworks/{multiarch}" + + +def package_version(prefix_path: Path) -> str: + """Extract the Python version being built from patchlevel.h.""" + for path in prefix_path.glob("**/patchlevel.h"): + text = path.read_text(encoding="utf-8") + if match := re.search( + r'\n\s*#define\s+PY_VERSION\s+"(.+)"\s*\n', text + ): + version = match[1] + # If not building against a tagged commit, add a timestamp to the + # version. Follow the PyPA version number rules, as this will make + # it easier to process with other tools. The version will have a + # `+` suffix once any official release has been made; a freshly + # forked main branch will have a version of 3.X.0a0. + if version.endswith("a0"): + version += "+" + if version.endswith("+"): + version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") + + return version + + sys.exit("Unable to determine Python version being packaged.") + + +def lib_platform_files(dirname, names): + """A file filter that ignores platform-specific files in the lib directory. + """ + path = Path(dirname) + if ( + path.parts[-3] == "lib" + and path.parts[-2].startswith("python") + and path.parts[-1] == "lib-dynload" + ): + return names + elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"): + ignored_names = set( + name + for name in names + if ( + name.startswith("_sysconfigdata_") + or name.startswith("_sysconfig_vars_") + or name == "build-details.json" + ) + ) + else: + ignored_names = set() + + return ignored_names + + +def lib_non_platform_files(dirname, names): + """A file filter that ignores anything *except* platform-specific files + in the lib directory. + """ + path = Path(dirname) + if path.parts[-2] == "lib" and path.parts[-1].startswith("python"): + return set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + else: + return set() + + +def create_xcframework(platform: str) -> str: + """Build an XCframework from the component parts for the platform. + + :return: The version number of the Python verion that was packaged. + """ + package_path = CROSS_BUILD_DIR / platform + try: + package_path.mkdir() + except FileExistsError: + raise RuntimeError( + f"{platform} XCframework already exists; do you need to run with --clean?" + ) from None + + frameworks = [] + # Merge Frameworks for each component SDK. If there's only one architecture + # for the SDK, we can use the compiled Python.framework as-is. However, if + # there's more than architecture, we need to merge the individual built + # frameworks into a merged "fat" framework. + for slice_name, slice_parts in HOSTS[platform].items(): + # Some parts are the same across all slices, so we use can any of the + # host frameworks as the source for the merged version. Use the first + # one on the list, as it's as representative as any other. + first_host_triple, first_multiarch = next(iter(slice_parts.items())) + first_framework = ( + framework_path(first_host_triple, first_multiarch) + / "Python.framework" + ) + + if len(slice_parts) == 1: + # The first framework is the only framework, so copy it. + print(f"Copying framework for {slice_name}...") + frameworks.append(first_framework) + else: + print(f"Merging framework for {slice_name}...") + slice_path = CROSS_BUILD_DIR / slice_name + slice_framework = slice_path / "Python.framework" + slice_framework.mkdir(exist_ok=True, parents=True) + + # Copy the Info.plist + shutil.copy( + first_framework / "Info.plist", + slice_framework / "Info.plist", + ) + + # Copy the headers + shutil.copytree( + first_framework / "Headers", + slice_framework / "Headers", + ) + + # Create the "fat" library binary for the slice + run( + ["lipo", "-create", "-output", slice_framework / "Python"] + + [ + ( + framework_path(host_triple, multiarch) + / "Python.framework/Python" + ) + for host_triple, multiarch in slice_parts.items() + ] + ) + + # Add this merged slice to the list to be added to the XCframework + frameworks.append(slice_framework) + + print() + print("Build XCframework...") + cmd = [ + "xcodebuild", + "-create-xcframework", + "-output", + package_path / "Python.xcframework", + ] + for framework in frameworks: + cmd.extend(["-framework", framework]) + + run(cmd) + + # Extract the package version from the merged framework + version = package_version(package_path / "Python.xcframework") + + # On non-macOS platforms, each framework in XCframework only contains the + # headers, libPython, plus an Info.plist. Other resources like the standard + # library and binary shims aren't allowed to live in framework; they need + # to be copied in separately. + print() + print("Copy additional resources...") + has_common_stdlib = False + for slice_name, slice_parts in HOSTS[platform].items(): + # Some parts are the same across all slices, so we can any of the + # host frameworks as the source for the merged version. + first_host_triple, first_multiarch = next(iter(slice_parts.items())) + first_path = framework_path(first_host_triple, first_multiarch) + first_framework = first_path / "Python.framework" + + slice_path = package_path / f"Python.xcframework/{slice_name}" + slice_framework = slice_path / "Python.framework" + + # Copy the binary helpers + print(f" - {slice_name} binaries") + shutil.copytree(first_path / "bin", slice_path / "bin") + + # Copy the include path (this will be a symlink to the framework headers) + print(f" - {slice_name} include files") + shutil.copytree( + first_path / "include", + slice_path / "include", + symlinks=True, + ) + + # Copy in the cross-architecture pyconfig.h + shutil.copy( + PYTHON_DIR / f"Apple/{platform}/Resources/pyconfig.h", + slice_framework / "Headers/pyconfig.h", + ) + + print(f" - {slice_name} architecture-specific files") + for host_triple, multiarch in slice_parts.items(): + print(f" - {multiarch} standard library") + arch, _ = multiarch.split("-", 1) + + if not has_common_stdlib: + print(" - using this architecture as the common stdlib") + shutil.copytree( + framework_path(host_triple, multiarch) / "lib", + package_path / "Python.xcframework/lib", + ignore=lib_platform_files, + ) + has_common_stdlib = True + + shutil.copytree( + framework_path(host_triple, multiarch) / "lib", + slice_path / f"lib-{arch}", + ignore=lib_non_platform_files, + ) + + # Copy the host's pyconfig.h to an architecture-specific name. + arch = multiarch.split("-")[0] + host_path = ( + CROSS_BUILD_DIR + / host_triple + / "Apple/iOS/Frameworks" + / multiarch + ) + host_framework = host_path / "Python.framework" + shutil.copy( + host_framework / "Headers/pyconfig.h", + slice_framework / f"Headers/pyconfig-{arch}.h", + ) + + print(" - build tools") + shutil.copytree( + PYTHON_DIR / "Apple/testbed/Python.xcframework/build", + package_path / "Python.xcframework/build", + ) + + return version + + +def package(context: argparse.Namespace) -> None: + """The implementation of the "package" command.""" + if context.clean: + clean(context, "package") + + with group("Building package"): + # Create an XCframework + version = create_xcframework(context.platform) + + # Clone testbed + print() + run( + [ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + CROSS_BUILD_DIR / context.platform / "Python.xcframework", + CROSS_BUILD_DIR / context.platform / "testbed", + ] + ) + + # Build the final archive + archive_name = ( + CROSS_BUILD_DIR + / "dist" + / f"python-{version}-{context.platform}-XCframework" + ) + + print() + print("Create package archive...") + shutil.make_archive( + str(CROSS_BUILD_DIR / archive_name), + format="gztar", + root_dir=CROSS_BUILD_DIR / context.platform, + base_dir=".", + ) + print() + print(f"{archive_name.relative_to(PYTHON_DIR)}.tar.gz created.") + + +def build(context: argparse.Namespace, host: str | None = None) -> None: + """The implementation of the "build" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, host) + + if host in {"all", "build"}: + for step in [ + configure_build_python, + make_build_python, + ]: + step(context) + + if host == "build": + hosts = [] + elif host in {"all", "hosts"}: + hosts = all_host_triples(context.platform) + else: + hosts = [host] + + for step_host in hosts: + for step in [ + configure_host_python, + make_host_python, + ]: + step(context, host=step_host) + + if host in {"all", "hosts"}: + package(context) + + +def test(context: argparse.Namespace, host: str | None = None) -> None: + """The implementation of the "test" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, "test") + + with group(f"Test {'XCframework' if host in {'all', 'hosts'} else host}"): + timestamp = str(time.time_ns())[:-6] + testbed_dir = ( + CROSS_BUILD_DIR / f"{context.platform}-testbed.{timestamp}" + ) + if host in {"all", "hosts"}: + framework_path = ( + CROSS_BUILD_DIR / context.platform / "Python.xcframework" + ) + else: + build_arch = platform.machine() + host_arch = host.split("-")[0] + + if not host.endswith("-simulator"): + print("Skipping test suite non-simulator build.") + return + elif build_arch != host_arch: + print( + f"Skipping test suite for an {host_arch} build " + f"on an {build_arch} machine." + ) + return + else: + framework_path = ( + CROSS_BUILD_DIR + / host + / f"Apple/{context.platform}" + / f"Frameworks/{apple_multiarch(host)}" + ) + + run( + [ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + framework_path, + testbed_dir, + ] + ) + + run( + [ + sys.executable, + testbed_dir, + "run", + "--verbose", + ] + + ( + ["--simulator", str(context.simulator)] + if context.simulator + else [] + ) + + [ + "--", + "test", + "--slow-ci" if context.slow else "--fast-ci", + "--single-process", + "--no-randomize", + # Timeout handling requires subprocesses; explicitly setting + # the timeout to -1 disables the faulthandler. + "--timeout=-1", + # Adding Python options requires the use of a subprocess to + # start a new Python interpreter. + "--dont-add-python-opts", + ] + ) + + +def ci(context: argparse.Namespace) -> None: + """The implementation of the "ci" command.""" + clean(context, "all") + build(context, host="all") + test(context, host="all") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "A tool for managing the build, package and test process of " + "CPython on Apple platforms." + ), + ) + parser.suggest_on_error = True + subcommands = parser.add_subparsers(dest="subcommand", required=True) + + clean = subcommands.add_parser( + "clean", + help="Delete all build directories", + ) + + configure_build = subcommands.add_parser( + "configure-build", help="Run `configure` for the build Python" + ) + subcommands.add_parser( + "make-build", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for a specific platform and target", + ) + make_host = subcommands.add_parser( + "make-host", + help="Run `make` for a specific platform and target", + ) + package = subcommands.add_parser( + "package", + help="Create a release package for the platform", + ) + build = subcommands.add_parser( + "build", + help="Build all platform targets and create the XCframework", + ) + test = subcommands.add_parser( + "test", + help="Run the testbed for a specific platform", + ) + ci = subcommands.add_parser( + "ci", + help="Run build, package, and test", + ) + + # platform argument + for cmd in [clean, configure_host, make_host, package, build, test, ci]: + cmd.add_argument( + "platform", + choices=HOSTS.keys(), + help="The target platform to build", + ) + + # host triple argument + for cmd in [configure_host, make_host]: + cmd.add_argument( + "host", + help="The host triple to build (e.g., arm64-apple-ios-simulator)", + ) + # optional host triple argument + for cmd in [clean, build, test]: + cmd.add_argument( + "host", + nargs="?", + default="all", + help=( + "The host triple to build (e.g., arm64-apple-ios-simulator), " + "or 'build' for just the build platform, or 'hosts' for all " + "host platforms, or 'all' for the build platform and all " + "hosts. Defaults to 'all'" + ), + ) + + # --clean option + for cmd in [configure_build, configure_host, build, package, test, ci]: + cmd.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete the relevant build directories first", + ) + + # --cache-dir option + for cmd in [configure_host, build, ci]: + cmd.add_argument( + "--cache-dir", + default="./cross-build/downloads", + help="The directory to store cached downloads.", + ) + + # --simulator option + for cmd in [test, ci]: + cmd.add_argument( + "--simulator", + help=( + "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to " + "the most recently released 'entry level' iPhone device. Device " + "architecture and OS version can also be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on " + "an ARM64 iPhone 16 Pro simulator running iOS 26.0." + ), + ) + cmd.add_argument( + "--slow", + action="store_true", + help="Run tests with --slow-ci options.", + ) + + for subcommand in [configure_build, configure_host, build, ci]: + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + + return parser.parse_args() + + +def print_called_process_error(e: subprocess.CalledProcessError) -> None: + for stream_name in ["stdout", "stderr"]: + content = getattr(e, stream_name) + stream = getattr(sys, stream_name) + if content: + stream.write(content) + if not content.endswith("\n"): + stream.write("\n") + + # shlex uses single quotes, so we surround the command with double quotes. + print( + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' + ) + + +def main() -> None: + # Handle SIGTERM the same way as SIGINT. This ensures that if we're + # terminated by the buildbot worker, we'll make an attempt to clean up our + # subprocesses. + def signal_handler(*args): + os.kill(os.getpid(), signal.SIGINT) + + signal.signal(signal.SIGTERM, signal_handler) + + # Process command line arguments + context = parse_args() + dispatch: dict[str, Callable] = { + "clean": clean, + "configure-build": configure_build_python, + "make-build": make_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "package": package, + "build": build, + "test": test, + "ci": ci, + } + + try: + dispatch[context.subcommand](context) + except CalledProcessError as e: + print() + print_called_process_error(e) + sys.exit(1) + except RuntimeError as e: + print() + print(e) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/Apple/iOS/README.md b/Apple/iOS/README.md new file mode 100644 index 00000000000000..124a05657aae09 --- /dev/null +++ b/Apple/iOS/README.md @@ -0,0 +1,328 @@ +# Python on iOS README + +**iOS support is [tier 3](https://peps.python.org/pep-0011/#tier-3).** + +This document provides a quick overview of some iOS specific features in the +Python distribution. + +These instructions are only needed if you're planning to compile Python for iOS +yourself. Most users should *not* need to do this. If you're looking to +experiment with writing an iOS app in Python, tools such as [BeeWare's +Briefcase](https://briefcase.readthedocs.io) and [Kivy's +Buildozer](https://buildozer.readthedocs.io) will provide a much more +approachable user experience. + +## Compilers for building on iOS + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting an open the Platforms tab of the Xcode Settings panel. + +## Building Python on iOS + +### ABIs and Architectures + +iOS apps can be deployed on physical devices, and on the iOS simulator. Although +the API used on these devices is identical, the ABI is different - you need to +link against different libraries for an iOS device build (`iphoneos`) or an +iOS simulator build (`iphonesimulator`). + +Apple uses the `XCframework` format to allow specifying a single dependency +that supports multiple ABIs. An `XCframework` is a wrapper around multiple +ABI-specific frameworks that share a common API. + +iOS can also support different CPU architectures within each ABI. At present, +there is only a single supported architecture on physical devices - ARM64. +However, the *simulator* supports 2 architectures - ARM64 (for running on Apple +Silicon machines), and x86_64 (for running on older Intel-based machines). + +To support multiple CPU architectures on a single platform, Apple uses a "fat +binary" format - a single physical file that contains support for multiple +architectures. It is possible to compile and use a "thin" single architecture +version of a binary for testing purposes; however, the "thin" binary will not be +portable to machines using other architectures. + +### Building a multi-architecture iOS XCframework + +The `Apple` subfolder of the Python repository acts as a build script that +can be used to coordinate the compilation of a complete iOS XCframework. To use +it, run:: + + python Apple build iOS + +This will: + +* Configure and compile a version of Python to run on the build machine +* Download pre-compiled binary dependencies for each platform +* Configure and build a `Python.framework` for each required architecture and + iOS SDK +* Merge the multiple `Python.framework` folders into a single `Python.xcframework` +* Produce a `.tar.gz` archive in the `cross-build/dist` folder containing + the `Python.xcframework`, plus a copy of the Testbed app pre-configured to + use the XCframework. + +The `Apple` build script has other entry points that will perform the +individual parts of the overall `build` target, plus targets to test the +build, clean the `cross-build` folder of iOS build products, and perform a +complete "build and test" CI run. The `--clean` flag can also be used on +individual commands to ensure that a stale build product are removed before +building. + +### Building a single-architecture framework + +If you're using the `Apple` build script, you won't need to build +individual frameworks. However, if you do need to manually configure an iOS +Python build for a single framework, the following options are available. + +#### iOS specific arguments to configure + +* `--enable-framework[=DIR]` + + This argument specifies the location where the Python.framework will be + installed. If `DIR` is not specified, the framework will be installed into + a subdirectory of the `iOS/Frameworks` folder. + + This argument *must* be provided when configuring iOS builds. iOS does not + support non-framework builds. + +* `--with-framework-name=NAME` + + Specify the name for the Python framework; defaults to `Python`. + + > [!NOTE] + > Unless you know what you're doing, changing the name of the Python + > framework on iOS is not advised. If you use this option, you won't be able + > to run the `Apple` build script without making significant manual + > alterations, and you won't be able to use any binary packages unless you + > compile them yourself using your own framework name. + +#### Building Python for iOS + +The Python build system will create a `Python.framework` that supports a +*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a +framework to contain non-library content, so the iOS build will produce a +`bin` and `lib` folder in the same output folder as `Python.framework`. +The `lib` folder will be needed at runtime to support the Python library. + +If you want to use Python in a real iOS project, you need to produce multiple +`Python.framework` builds, one for each ABI and architecture. iOS builds of +Python *must* be constructed as framework builds. To support this, you must +provide the `--enable-framework` flag when configuring the build. The build +also requires the use of cross-compilation. The minimal commands for building +Python for the ARM64 iOS simulator will look something like: +``` +export PATH="$(pwd)/Apple/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" +./configure \ + --enable-framework \ + --host=arm64-apple-ios-simulator \ + --build=arm64-apple-darwin \ + --with-build-python=/path/to/python.exe +make +make install +``` + +In this invocation: + +* `Apple/iOS/Resources/bin` has been added to the path, providing some shims for the + compilers and linkers needed by the build. Xcode requires the use of `xcrun` + to invoke compiler tooling. However, if `xcrun` is pre-evaluated and the + result passed to `configure`, these results can embed user- and + version-specific paths into the sysconfig data, which limits the portability + of the compiled Python. Alternatively, if `xcrun` is used *as* the compiler, + it requires that compiler variables like `CC` include spaces, which can + cause significant problems with many C configuration systems which assume that + `CC` will be a single executable. + + To work around this problem, the `Apple/iOS/Resources/bin` folder contains some + wrapper scripts that present as simple compilers and linkers, but wrap + underlying calls to `xcrun`. This allows configure to use a `CC` + definition without spaces, and without user- or version-specific paths, while + retaining the ability to adapt to the local Xcode install. These scripts are + included in the `bin` directory of an iOS install. + + These scripts will, by default, use the currently active Xcode installation. + If you want to use a different Xcode installation, you can use + `xcode-select` to set a new default Xcode globally, or you can use the + `DEVELOPER_DIR` environment variable to specify an Xcode install. The + scripts will use the default `iphoneos`/`iphonesimulator` SDK version for + the select Xcode install; if you want to use a different SDK, you can set the + `IOS_SDK_VERSION` environment variable. (e.g, setting + `IOS_SDK_VERSION=17.1` would cause the scripts to use the `iphoneos17.1` + and `iphonesimulator17.1` SDKs, regardless of the Xcode default.) + + The path has also been cleared of any user customizations. A common source of + bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS + build. Resetting the path to a known "bare bones" value is the easiest way to + avoid these problems. + +* `--host` is the architecture and ABI that you want to build, in GNU compiler + triple format. This will be one of: + + - `arm64-apple-ios` for ARM64 iOS devices. + - `arm64-apple-ios-simulator` for the iOS simulator running on Apple + Silicon devices. + - `x86_64-apple-ios-simulator` for the iOS simulator running on Intel + devices. + +* `--build` is the GNU compiler triple for the machine that will be running + the compiler. This is one of: + + - `arm64-apple-darwin` for Apple Silicon devices. + - `x86_64-apple-darwin` for Intel devices. + +* `/path/to/python.exe` is the path to a Python binary on the machine that + will be running the compiler. This is needed because the Python compilation + process involves running some Python code. On a normal desktop build of + Python, you can compile a python interpreter and then use that interpreter to + run Python code. However, the binaries produced for iOS won't run on macOS, so + you need to provide an external Python interpreter. This interpreter must be + the same version as the Python that is being compiled. To be completely safe, + this should be the *exact* same commit hash. However, the longer a Python + release has been stable, the more likely it is that this constraint can be + relaxed - the same micro version will often be sufficient. + +* The `install` target for iOS builds is slightly different to other + platforms. On most platforms, `make install` will install the build into + the final runtime location. This won't be the case for iOS, as the final + runtime location will be on a physical device. + + However, you still need to run the `install` target for iOS builds, as it + performs some final framework assembly steps. The location specified with + `--enable-framework` will be the location where `make install` will + assemble the complete iOS framework. This completed framework can then + be copied and relocated as required. + +For a full CPython build, you also need to specify the paths to iOS builds of +the binary libraries that CPython depends on (such as XZ, LibFFI and OpenSSL). +This can be done by defining library specific environment variables (such as +`LIBLZMA_CFLAGS`, `LIBLZMA_LIBS`), and the `--with-openssl` configure +option. Versions of these libraries pre-compiled for iOS can be found in [this +repository](https://github.com/beeware/cpython-apple-source-deps/releases). +LibFFI is especially important, as many parts of the standard library +(including the `platform`, `sysconfig` and `webbrowser` modules) require +the use of the `ctypes` module at runtime. + +By default, Python will be compiled with an iOS deployment target (i.e., the +minimum supported iOS version) of 13.0. To specify a different deployment +target, provide the version number as part of the `--host` argument - for +example, `--host=arm64-apple-ios15.4-simulator` would compile an ARM64 +simulator build with a deployment target of 15.4. + +## Testing Python on iOS + +### Testing a multi-architecture framework + +Once you have a built an XCframework, you can test that framework by running: + + $ python Apple test iOS + +### Testing a single-architecture framework + +The `Apple/testbed` folder that contains an Xcode project that is able to run +the Python test suite on Apple platforms. This project converts the Python test +suite into a single test case in Xcode's XCTest framework. The single XCTest +passes if the test suite passes. + +To run the test suite, configure a Python build for an iOS simulator (i.e., +`--host=arm64-apple-ios-simulator` or `--host=x86_64-apple-ios-simulator` +), specifying a framework build (i.e. `--enable-framework`). Ensure that your +`PATH` has been configured to include the `Apple/iOS/Resources/bin` folder and +exclude any non-iOS tools, then run: +``` +make all +make install +make testios +``` + +This will: + +* Build an iOS framework for your chosen architecture; +* Finalize the single-platform framework; +* Make a clean copy of the testbed project; +* Install the Python iOS framework into the copy of the testbed project; and +* Run the test suite on an "entry-level device" simulator (i.e., an iPhone SE, + iPhone 16e, or a similar). + +On success, the test suite will exit and report successful completion of the +test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 +minutes to run; a couple of extra minutes is required to compile the testbed +project, and then boot and prepare the iOS simulator. + +### Debugging test failures + +Running `python Apple test iOS` generates a standalone version of the +`Apple/testbed` project, and runs the full test suite. It does this using +`Apple/testbed` itself - the folder is an executable module that can be used +to create and run a clone of the testbed project. The standalone version of the +testbed will be created in a directory named +`cross-build/iOS-testbed.`. + +You can generate your own standalone testbed instance by running: +``` +python cross-build/iOS/testbed clone my-testbed +``` + +In this invocation, `my-testbed` is the name of the folder for the new +testbed clone. + +If you've built your own XCframework, or you only want to test a single architecture, +you can construct a standalone testbed instance by running: +``` +python Apple/testbed clone --platform iOS --framework my-testbed +``` + +The framework path can be the path path to a `Python.xcframework`, or the +path to a folder that contains a single-platform `Python.framework`. + +You can then use the `my-testbed` folder to run the Python test suite, +passing in any command line arguments you may require. For example, if you're +trying to diagnose a failure in the `os` module, you might run: +``` +python my-testbed run -- test -W test_os +``` + +This is the equivalent of running `python -m test -W test_os` on a desktop +Python build. Any arguments after the `--` will be passed to testbed as if +they were arguments to `python -m` on a desktop machine. + +### Testing in Xcode + +You can also open the testbed project in Xcode by running: +``` +open my-testbed/iOSTestbed.xcodeproj +``` + +This will allow you to use the full Xcode suite of tools for debugging. + +The arguments used to run the test suite are defined as part of the test plan. +To modify the test plan, select the test plan node of the project tree (it +should be the first child of the root node), and select the "Configurations" +tab. Modify the "Arguments Passed On Launch" value to change the testing +arguments. + +The test plan also disables parallel testing, and specifies the use of the +`Testbed.lldbinit` file for providing configuration of the debugger. The +default debugger configuration disables automatic breakpoints on the +`SIGINT`, `SIGUSR1`, `SIGUSR2`, and `SIGXFSZ` signals. + +### Testing on an iOS device + +To test on an iOS device, the app needs to be signed with known developer +credentials. To obtain these credentials, you must have an iOS Developer +account, and your Xcode install will need to be logged into your account (see +the Accounts tab of the Preferences dialog). + +Once the project is open, and you're signed into your Apple Developer account, +select the root node of the project tree (labeled "iOSTestbed"), then the +"Signing & Capabilities" tab in the details page. Select a development team +(this will likely be your own name), and plug in a physical device to your +macOS machine with a USB cable. You should then be able to select your physical +device from the list of targets in the pulldown in the Xcode titlebar. diff --git a/iOS/Resources/Info.plist.in b/Apple/iOS/Resources/Info.plist.in similarity index 100% rename from iOS/Resources/Info.plist.in rename to Apple/iOS/Resources/Info.plist.in diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-ar b/Apple/iOS/Resources/bin/arm64-apple-ios-ar new file mode 100755 index 00000000000000..3cf3eb218741fa --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang b/Apple/iOS/Resources/bin/arm64-apple-ios-clang new file mode 100755 index 00000000000000..f50d5b5142fc76 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ new file mode 100755 index 00000000000000..0794731d7dcbda --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-cpp b/Apple/iOS/Resources/bin/arm64-apple-ios-cpp new file mode 100755 index 00000000000000..24fa1506bab827 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} -E "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..b836b6db9025bb --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..4891a00876e0bd --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..58b2a5f6f18c2b --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..c9df94e8b7c837 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip new file mode 100755 index 00000000000000..fd59d309b73a20 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-strip b/Apple/iOS/Resources/bin/arm64-apple-ios-strip new file mode 100755 index 00000000000000..75e823a3d02d61 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..b836b6db9025bb --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..f4739a7b945d01 --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..c348ae4c10395b --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..6d7f8084c9fdcc --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip new file mode 100755 index 00000000000000..c5cfb28929195a --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch x86_64 "$@" diff --git a/iOS/Resources/pyconfig.h b/Apple/iOS/Resources/pyconfig.h similarity index 100% rename from iOS/Resources/pyconfig.h rename to Apple/iOS/Resources/pyconfig.h diff --git a/iOS/testbed/Python.xcframework/Info.plist b/Apple/testbed/Python.xcframework/Info.plist similarity index 100% rename from iOS/testbed/Python.xcframework/Info.plist rename to Apple/testbed/Python.xcframework/Info.plist diff --git a/iOS/testbed/iOSTestbed/dylib-Info-template.plist b/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist similarity index 96% rename from iOS/testbed/iOSTestbed/dylib-Info-template.plist rename to Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist index f652e272f71c88..d6caa01c1e44b9 100644 --- a/iOS/testbed/iOSTestbed/dylib-Info-template.plist +++ b/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist @@ -19,7 +19,7 @@ iPhoneOS MinimumOSVersion - 12.0 + 13.0 CFBundleVersion 1 diff --git a/Apple/testbed/Python.xcframework/build/utils.sh b/Apple/testbed/Python.xcframework/build/utils.sh new file mode 100755 index 00000000000000..9cfe74720f26d4 --- /dev/null +++ b/Apple/testbed/Python.xcframework/build/utils.sh @@ -0,0 +1,137 @@ +# Utility methods for use in an Xcode project. +# +# An iOS XCframework cannot include any content other than the library binary +# and relevant metadata. However, Python requires a standard library at runtime. +# Therefore, it is necessary to add a build step to an Xcode app target that +# processes the standard library and puts the content into the final app. +# +# In general, these tools will be invoked after bundle resources have been +# copied into the app, but before framework embedding (and signing). +# +# The following is an example script, assuming that: +# * Python.xcframework is in the root of the project +# * There is an `app` folder that contains the app code +# * There is an `app_packages` folder that contains installed Python packages. +# ----- +# set -e +# source $PROJECT_DIR/Python.xcframework/build/build_utils.sh +# install_python Python.xcframework app app_packages +# ----- + +# Copy the standard library from the XCframework into the app bundle. +# +# Accepts one argument: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +install_stdlib() { + PYTHON_XCFRAMEWORK_PATH=$1 + + mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" + if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then + echo "Installing Python modules for iOS Simulator" + if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/ios-arm64-simulator" ]; then + SLICE_FOLDER="ios-arm64-simulator" + else + SLICE_FOLDER="ios-arm64_x86_64-simulator" + fi + else + echo "Installing Python modules for iOS Device" + SLICE_FOLDER="ios-arm64" + fi + + # If the XCframework has a shared lib folder, then it's a full framework. + # Copy both the common and slice-specific part of the lib directory. + # Otherwise, it's a single-arch framework; use the "full" lib folder. + if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib" ]; then + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/" + else + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + fi +} + +# Convert a single .so library into a framework that iOS can load. +# +# Accepts three arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +# 2. The base path, relative to the installed location in the app bundle, that +# needs to be processed. Any .so file found in this path (or a subdirectory +# of it) will be processed. +# 2. The full path to a single .so file to process. This path should include +# the base path. +install_dylib () { + PYTHON_XCFRAMEWORK_PATH=$1 + INSTALL_BASE=$2 + FULL_EXT=$3 + + # The name of the extension file + EXT=$(basename "$FULL_EXT") + # The location of the extension file, relative to the bundle + RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} + # The path to the extension file, relative to the install base + PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} + # The full dotted name of the extension module, constructed from the file path. + FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); + # A bundle identifier; not actually used, but required by Xcode framework packaging + FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") + # The name of the framework folder. + FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" + + # If the framework folder doesn't exist, create it. + if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then + echo "Creating framework for $RELATIVE_EXT" + mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" + cp "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/build/$PLATFORM_FAMILY_NAME-dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + fi + + echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + # Create a placeholder .fwork file where the .so was + echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork + # Create a back reference to the .so file location in the framework + echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" + + echo "Signing framework as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." + /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" +} + +# Process all the dynamic libraries in a path into Framework format. +# +# Accepts two arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +# 2. The base path, relative to the installed location in the app bundle, that +# needs to be processed. Any .so file found in this path (or a subdirectory +# of it) will be processed. +process_dylibs () { + PYTHON_XCFRAMEWORK_PATH=$1 + LIB_PATH=$2 + find "$CODESIGNING_FOLDER_PATH/$LIB_PATH" -name "*.so" | while read FULL_EXT; do + install_dylib $PYTHON_XCFRAMEWORK_PATH "$LIB_PATH/" "$FULL_EXT" + done +} + +# The entry point for post-processing a Python XCframework. +# +# Accepts 1 or more arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. If the XCframework is in the root of the project, +# 2+. The path of a package, relative to the root of the packaged app, that contains +# library content that should be processed for binary libraries. +install_python() { + PYTHON_XCFRAMEWORK_PATH=$1 + shift + + install_stdlib $PYTHON_XCFRAMEWORK_PATH + PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") + echo "Install Python $PYTHON_VER standard library extension modules..." + process_dylibs $PYTHON_XCFRAMEWORK_PATH python/lib/$PYTHON_VER/lib-dynload + + for package_path in $@; do + echo "Installing $package_path extension modules ..." + process_dylibs $PYTHON_XCFRAMEWORK_PATH $package_path + done +} diff --git a/iOS/testbed/Python.xcframework/ios-arm64/README b/Apple/testbed/Python.xcframework/ios-arm64/README similarity index 100% rename from iOS/testbed/Python.xcframework/ios-arm64/README rename to Apple/testbed/Python.xcframework/ios-arm64/README diff --git a/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README b/Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README similarity index 100% rename from iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README rename to Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README diff --git a/iOS/testbed/iOSTestbed.lldbinit b/Apple/testbed/Testbed.lldbinit similarity index 100% rename from iOS/testbed/iOSTestbed.lldbinit rename to Apple/testbed/Testbed.lldbinit diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/Apple/testbed/TestbedTests/TestbedTests.m similarity index 97% rename from iOS/testbed/iOSTestbedTests/iOSTestbedTests.m rename to Apple/testbed/TestbedTests/TestbedTests.m index d3159f5c2e155c..80741097e4c80d 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/Apple/testbed/TestbedTests/TestbedTests.m @@ -1,11 +1,11 @@ #import #import -@interface iOSTestbedTests : XCTestCase +@interface TestbedTests : XCTestCase @end -@implementation iOSTestbedTests +@implementation TestbedTests - (void)testPython { @@ -41,14 +41,14 @@ - (void)testPython { // The processInfo arguments contain the binary that is running, // followed by the arguments defined in the test plan. This means: // run_module = test_args[1] - // argv = ["iOSTestbed"] + test_args[2:] + // argv = ["Testbed"] + test_args[2:] test_args = [[NSProcessInfo processInfo] arguments]; if (test_args == NULL) { NSLog(@"Unable to identify test arguments."); } NSLog(@"Test arguments: %@", test_args); argv = malloc(sizeof(char *) * ([test_args count] - 1)); - argv[0] = "iOSTestbed"; + argv[0] = "Testbed"; for (int i = 1; i < [test_args count] - 1; i++) { argv[i] = [[test_args objectAtIndex:i+1] UTF8String]; } diff --git a/iOS/testbed/__main__.py b/Apple/testbed/__main__.py similarity index 59% rename from iOS/testbed/__main__.py rename to Apple/testbed/__main__.py index 6a4d9c76d162b4..4a1333380cdb6d 100644 --- a/iOS/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -6,6 +6,9 @@ import sys from pathlib import Path +TEST_SLICES = { + "iOS": "ios-arm64_x86_64-simulator", +} DECODE_ARGS = ("UTF-8", "backslashreplace") @@ -21,45 +24,49 @@ # Select a simulator device to use. -def select_simulator_device(): +def select_simulator_device(platform): # List the testing simulators, in JSON format raw_json = subprocess.check_output(["xcrun", "simctl", "list", "-j"]) json_data = json.loads(raw_json) - # Any device will do; we'll look for "SE" devices - but the name isn't - # consistent over time. Older Xcode versions will use "iPhone SE (Nth - # generation)"; As of 2025, they've started using "iPhone 16e". - # - # When Xcode is updated after a new release, new devices will be available - # and old ones will be dropped from the set available on the latest iOS - # version. Select the one with the highest minimum runtime version - this - # is an indicator of the "newest" released device, which should always be - # supported on the "most recent" iOS version. - se_simulators = sorted( - (devicetype["minRuntimeVersion"], devicetype["name"]) - for devicetype in json_data["devicetypes"] - if devicetype["productFamily"] == "iPhone" - and ( - ( - "iPhone " in devicetype["name"] - and devicetype["name"].endswith("e") + if platform == "iOS": + # Any iOS device will do; we'll look for "SE" devices - but the name isn't + # consistent over time. Older Xcode versions will use "iPhone SE (Nth + # generation)"; As of 2025, they've started using "iPhone 16e". + # + # When Xcode is updated after a new release, new devices will be available + # and old ones will be dropped from the set available on the latest iOS + # version. Select the one with the highest minimum runtime version - this + # is an indicator of the "newest" released device, which should always be + # supported on the "most recent" iOS version. + se_simulators = sorted( + (devicetype["minRuntimeVersion"], devicetype["name"]) + for devicetype in json_data["devicetypes"] + if devicetype["productFamily"] == "iPhone" + and ( + ( + "iPhone " in devicetype["name"] + and devicetype["name"].endswith("e") + ) + or "iPhone SE " in devicetype["name"] ) - or "iPhone SE " in devicetype["name"] ) - ) + simulator = se_simulators[-1][1] + else: + raise ValueError(f"Unknown platform {platform}") - return se_simulators[-1][1] + return simulator -def xcode_test(location, simulator, verbose): +def xcode_test(location: Path, platform: str, simulator: str, verbose: bool): # Build and run the test suite on the named simulator. args = [ "-project", - str(location / "iOSTestbed.xcodeproj"), + str(location / f"{platform}Testbed.xcodeproj"), "-scheme", - "iOSTestbed", + f"{platform}Testbed", "-destination", - f"platform=iOS Simulator,name={simulator}", + f"platform={platform} Simulator,name={simulator}", "-derivedDataPath", str(location / "DerivedData"), ] @@ -89,10 +96,24 @@ def xcode_test(location, simulator, verbose): exit(status) +def copy(src, tgt): + """An all-purpose copy. + + If src is a file, it is copied. If src is a symlink, it is copied *as a + symlink*. If src is a directory, the full tree is duplicated, with symlinks + being preserved. + """ + if src.is_file() or src.is_symlink(): + shutil.copyfile(src, tgt, follow_symlinks=False) + else: + shutil.copytree(src, tgt, symlinks=True) + + def clone_testbed( source: Path, target: Path, framework: Path, + platform: str, apps: list[Path], ) -> None: if target.exists(): @@ -101,11 +122,11 @@ def clone_testbed( if framework is None: if not ( - source / "Python.xcframework/ios-arm64_x86_64-simulator/bin" + source / "Python.xcframework" / TEST_SLICES[platform] / "bin" ).is_dir(): print( f"The testbed being cloned ({source}) does not contain " - f"a simulator framework. Re-run with --framework" + "a framework with slices. Re-run with --framework" ) sys.exit(11) else: @@ -124,33 +145,49 @@ def clone_testbed( print("Cloning testbed project:") print(f" Cloning {source}...", end="") - shutil.copytree(source, target, symlinks=True) + # Only copy the files for the platform being cloned plus the files common + # to all platforms. The XCframework will be copied later, if needed. + target.mkdir(parents=True) + + for name in [ + "__main__.py", + "TestbedTests", + "Testbed.lldbinit", + f"{platform}Testbed", + f"{platform}Testbed.xcodeproj", + f"{platform}Testbed.xctestplan", + ]: + copy(source / name, target / name) + print(" done") + orig_xc_framework_path = source / "Python.xcframework" xc_framework_path = target / "Python.xcframework" - sim_framework_path = xc_framework_path / "ios-arm64_x86_64-simulator" + test_framework_path = xc_framework_path / TEST_SLICES[platform] if framework is not None: if framework.suffix == ".xcframework": print(" Installing XCFramework...", end="") - if xc_framework_path.is_dir(): - shutil.rmtree(xc_framework_path) - else: - xc_framework_path.unlink(missing_ok=True) xc_framework_path.symlink_to( framework.relative_to(xc_framework_path.parent, walk_up=True) ) print(" done") else: print(" Installing simulator framework...", end="") - if sim_framework_path.is_dir(): - shutil.rmtree(sim_framework_path) + # We're only installing a slice of a framework; we need + # to do a full tree copy to make sure we don't damage + # symlinked content. + shutil.copytree(orig_xc_framework_path, xc_framework_path) + if test_framework_path.is_dir(): + shutil.rmtree(test_framework_path) else: - sim_framework_path.unlink(missing_ok=True) - sim_framework_path.symlink_to( - framework.relative_to(sim_framework_path.parent, walk_up=True) + test_framework_path.unlink(missing_ok=True) + test_framework_path.symlink_to( + framework.relative_to(test_framework_path.parent, walk_up=True) ) print(" done") else: + copy(orig_xc_framework_path, xc_framework_path) + if ( xc_framework_path.is_symlink() and not xc_framework_path.readlink().is_absolute() @@ -158,39 +195,39 @@ def clone_testbed( # XCFramework is a relative symlink. Rewrite the symlink relative # to the new location. print(" Rewriting symlink to XCframework...", end="") - orig_xc_framework_path = ( + resolved_xc_framework_path = ( source / xc_framework_path.readlink() ).resolve() xc_framework_path.unlink() xc_framework_path.symlink_to( - orig_xc_framework_path.relative_to( + resolved_xc_framework_path.relative_to( xc_framework_path.parent, walk_up=True ) ) print(" done") elif ( - sim_framework_path.is_symlink() - and not sim_framework_path.readlink().is_absolute() + test_framework_path.is_symlink() + and not test_framework_path.readlink().is_absolute() ): print(" Rewriting symlink to simulator framework...", end="") # Simulator framework is a relative symlink. Rewrite the symlink # relative to the new location. - orig_sim_framework_path = ( - source / "Python.XCframework" / sim_framework_path.readlink() + orig_test_framework_path = ( + source / "Python.XCframework" / test_framework_path.readlink() ).resolve() - sim_framework_path.unlink() - sim_framework_path.symlink_to( - orig_sim_framework_path.relative_to( - sim_framework_path.parent, walk_up=True + test_framework_path.unlink() + test_framework_path.symlink_to( + orig_test_framework_path.relative_to( + test_framework_path.parent, walk_up=True ) ) print(" done") else: - print(" Using pre-existing iOS framework.") + print(" Using pre-existing Python framework.") for app_src in apps: print(f" Installing app {app_src.name!r}...", end="") - app_target = target / f"iOSTestbed/app/{app_src.name}" + app_target = target / f"Testbed/app/{app_src.name}" if app_target.is_dir(): shutil.rmtree(app_target) shutil.copytree(app_src, app_target) @@ -199,9 +236,9 @@ def clone_testbed( print(f"Successfully cloned testbed: {target.resolve()}") -def update_test_plan(testbed_path, args): +def update_test_plan(testbed_path, platform, args): # Modify the test plan to use the requested test arguments. - test_plan_path = testbed_path / "iOSTestbed.xctestplan" + test_plan_path = testbed_path / f"{platform}Testbed.xctestplan" with test_plan_path.open("r", encoding="utf-8") as f: test_plan = json.load(f) @@ -213,32 +250,50 @@ def update_test_plan(testbed_path, args): json.dump(test_plan, f, indent=2) -def run_testbed(simulator: str | None, args: list[str], verbose: bool = False): +def run_testbed( + platform: str, + simulator: str | None, + args: list[str], + verbose: bool = False, +): location = Path(__file__).parent print("Updating test plan...", end="") - update_test_plan(location, args) + update_test_plan(location, platform, args) print(" done.") if simulator is None: - simulator = select_simulator_device() + simulator = select_simulator_device(platform) print(f"Running test on {simulator}") - xcode_test(location, simulator=simulator, verbose=verbose) + xcode_test( + location, + platform=platform, + simulator=simulator, + verbose=verbose, + ) def main(): + # Look for directories like `iOSTestbed` as an indicator of the platforms + # that the testbed folder supports. The original source testbed can support + # many platforms, but when cloned, only one platform is preserved. + available_platforms = [ + platform + for platform in ["iOS"] + if (Path(__file__).parent / f"{platform}Testbed").is_dir() + ] + parser = argparse.ArgumentParser( description=( - "Manages the process of testing a Python project in the iOS simulator." + "Manages the process of testing an Apple Python project through Xcode." ), ) subcommands = parser.add_subparsers(dest="subcommand") - clone = subcommands.add_parser( "clone", description=( - "Clone the testbed project, copying in an iOS Python framework and" + "Clone the testbed project, copying in a Python framework and" "any specified application code." ), help="Clone a testbed project to a new location.", @@ -250,6 +305,13 @@ def main(): "XCFramework) to use when running the testbed" ), ) + clone.add_argument( + "--platform", + dest="platform", + choices=available_platforms, + default=available_platforms[0], + help=f"The platform to target (default: {available_platforms[0]})", + ) clone.add_argument( "--app", dest="apps", @@ -272,6 +334,13 @@ def main(): ), help="Run a testbed project", ) + run.add_argument( + "--platform", + dest="platform", + choices=available_platforms, + default=available_platforms[0], + help=f"The platform to target (default: {available_platforms[0]})", + ) run.add_argument( "--simulator", help=( @@ -306,22 +375,26 @@ def main(): framework=Path(context.framework).resolve() if context.framework else None, + platform=context.platform, apps=[Path(app) for app in context.apps], ) elif context.subcommand == "run": if test_args: if not ( Path(__file__).parent - / "Python.xcframework/ios-arm64_x86_64-simulator/bin" + / "Python.xcframework" + / TEST_SLICES[context.platform] + / "bin" ).is_dir(): print( - f"Testbed does not contain a compiled iOS framework. Use " + f"Testbed does not contain a compiled Python framework. Use " f"`python {sys.argv[0]} clone ...` to create a runnable " f"clone of this testbed." ) sys.exit(20) run_testbed( + platform=context.platform, simulator=context.simulator, verbose=context.verbose, args=test_args, diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj similarity index 79% rename from iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj rename to Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj index 18cdafd8127520..f8835a3bc587df 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -11,12 +11,11 @@ 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 607A66212B0EFA390010BFC8 /* Assets.xcassets */; }; 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */; }; 607A66282B0EFA390010BFC8 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66272B0EFA390010BFC8 /* main.m */; }; - 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */; }; + 607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */; }; 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; - 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */ = {isa = PBXBuildFile; fileRef = 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */; }; 608619542CB77BA900F46182 /* app_packages in Resources */ = {isa = PBXBuildFile; fileRef = 608619532CB77BA900F46182 /* app_packages */; }; 608619562CB7819B00F46182 /* app in Resources */ = {isa = PBXBuildFile; fileRef = 608619552CB7819B00F46182 /* app */; }; /* End PBXBuildFile section */ @@ -64,9 +63,8 @@ 607A66242B0EFA390010BFC8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 607A66272B0EFA390010BFC8 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = iOSTestbedTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; - 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = iOSTestbedTests.m; sourceTree = ""; }; + 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = TestbedTests.m; sourceTree = ""; }; 607A664A2B0EFB310010BFC8 /* Python.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Python.xcframework; sourceTree = ""; }; - 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "dylib-Info-template.plist"; sourceTree = ""; }; 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "iOSTestbed-Info.plist"; sourceTree = ""; }; 608619532CB77BA900F46182 /* app_packages */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app_packages; sourceTree = ""; }; 608619552CB7819B00F46182 /* app */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app; sourceTree = ""; }; @@ -99,7 +97,7 @@ 60FE0EFB2E56BB6D00524F87 /* iOSTestbed.xctestplan */, 607A664A2B0EFB310010BFC8 /* Python.xcframework */, 607A66142B0EFA380010BFC8 /* iOSTestbed */, - 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */, + 607A66302B0EFA3A0010BFC8 /* TestbedTests */, 607A66132B0EFA380010BFC8 /* Products */, 607A664F2B0EFFE00010BFC8 /* Frameworks */, ); @@ -120,7 +118,6 @@ 608619552CB7819B00F46182 /* app */, 608619532CB77BA900F46182 /* app_packages */, 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */, - 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */, 607A66152B0EFA380010BFC8 /* AppDelegate.h */, 607A66162B0EFA380010BFC8 /* AppDelegate.m */, 607A66212B0EFA390010BFC8 /* Assets.xcassets */, @@ -130,12 +127,12 @@ path = iOSTestbed; sourceTree = ""; }; - 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + 607A66302B0EFA3A0010BFC8 /* TestbedTests */ = { isa = PBXGroup; children = ( - 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */, + 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */, ); - path = iOSTestbedTests; + path = TestbedTests; sourceTree = ""; }; 607A664F2B0EFFE00010BFC8 /* Frameworks */ = { @@ -155,8 +152,7 @@ 607A660E2B0EFA380010BFC8 /* Sources */, 607A660F2B0EFA380010BFC8 /* Frameworks */, 607A66102B0EFA380010BFC8 /* Resources */, - 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */, - 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */, + 607A66552B0F061D0010BFC8 /* Process Python libraries */, 607A664E2B0EFC080010BFC8 /* Embed Frameworks */, ); buildRules = ( @@ -230,7 +226,6 @@ buildActionMask = 2147483647; files = ( 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */, - 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */, 608619562CB7819B00F46182 /* app in Resources */, 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */, 608619542CB77BA900F46182 /* app_packages in Resources */, @@ -247,7 +242,7 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ - 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */ = { + 607A66552B0F061D0010BFC8 /* Process Python libraries */ = { isa = PBXShellScriptBuildPhase; alwaysOutOfDate = 1; buildActionMask = 2147483647; @@ -257,34 +252,14 @@ ); inputPaths = ( ); - name = "Install Target Specific Python Standard Library"; + name = "Process Python libraries"; outputFileListPaths = ( ); outputPaths = ( ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; - showEnvVarsInLog = 0; - }; - 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { - isa = PBXShellScriptBuildPhase; - alwaysOutOfDate = 1; - buildActionMask = 2147483647; - files = ( - ); - inputFileListPaths = ( - ); - inputPaths = ( - ); - name = "Prepare Python Binary Modules"; - outputFileListPaths = ( - ); - outputPaths = ( - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + shellScript = "set -e\nsource $PROJECT_DIR/Python.xcframework/build/utils.sh\ninstall_python Python.xcframework app app_packages\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ @@ -303,7 +278,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */, + 607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/iOS/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme b/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme similarity index 97% rename from iOS/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme rename to Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme index d093a46f02e95d..3c330a4152bf92 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme +++ b/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme @@ -27,7 +27,7 @@ buildConfiguration = "Debug" selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" - customLLDBInitFile = "/Users/rkm/projects/pyspamsum/localtest/iOSTestbed.lldbinit" + customLLDBInitFile = "$(SOURCE_ROOT)/Testbed.lldbinit" shouldUseLaunchSchemeArgsEnv = "YES"> ${FULL_EXT%.so}.fwork - # Create a back reference to the .so file location in the framework - echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" - } - - PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") - echo "Install Python $PYTHON_VER standard library extension modules..." - find "$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload" -name "*.so" | while read FULL_EXT; do - install_dylib python/lib/$PYTHON_VER/lib-dynload/ "$FULL_EXT" - done - - # Clean up dylib template - rm -f "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" - - echo "Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." - find "$CODESIGNING_FOLDER_PATH/Frameworks" -name "*.framework" -exec /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "{}" \; - -10. Add Objective C code to initialize and use a Python interpreter in embedded - mode. You should ensure that: +8. Add Objective C code to initialize and use a Python interpreter in embedded + mode. You should ensure that: * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*; * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*; @@ -309,22 +244,19 @@ To add Python to an iOS Xcode project: Your app's bundle location can be determined using ``[[NSBundle mainBundle] resourcePath]``. -Steps 8, 9 and 10 of these instructions assume that you have a single folder of +Steps 7 and 8 of these instructions assume that you have a single folder of pure Python application code, named ``app``. If you have third-party binary modules in your app, some additional steps will be required: * You need to ensure that any folders containing third-party binaries are - either associated with the app target, or copied in as part of step 8. Step 8 - should also purge any binaries that are not appropriate for the platform a - specific build is targeting (i.e., delete any device binaries if you're - building an app targeting the simulator). + either associated with the app target, or are explicitly copied as part of + step 7. Step 7 should also purge any binaries that are not appropriate for + the platform a specific build is targeting (i.e., delete any device binaries + if you're building an app targeting the simulator). -* Any folders that contain third-party binaries must be processed into - framework form by step 9. The invocation of ``install_dylib`` that processes - the ``lib-dynload`` folder can be copied and adapted for this purpose. - -* If you're using a separate folder for third-party packages, ensure that folder - is included as part of the :envvar:`PYTHONPATH` configuration in step 10. +* If you're using a separate folder for third-party packages, ensure that + folder is added to the end of the call to ``install_python`` in step 7, and + as part of the :envvar:`PYTHONPATH` configuration in step 8. * If any of the folders that contain third-party packages will contain ``.pth`` files, you should add that folder as a *site directory* (using @@ -334,25 +266,30 @@ modules in your app, some additional steps will be required: Testing a Python package ------------------------ -The CPython source tree contains :source:`a testbed project ` that +The CPython source tree contains :source:`a testbed project ` that is used to run the CPython test suite on the iOS simulator. This testbed can also be used as a testbed project for running your Python library's test suite on iOS. -After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst` -for details), create a clone of the Python iOS testbed project by running: +After building or obtaining an iOS XCFramework (see :source:`Apple/iOS/README.md` +for details), create a clone of the Python iOS testbed project. If you used the +``Apple`` build script to build the XCframework, you can run: + +.. code-block:: bash + + $ python cross-build/iOS/testbed clone --app --app app-testbed + +Or, if you've sourced your own XCframework, by running: .. code-block:: bash - $ python iOS/testbed clone --framework --app --app app-testbed + $ python Apple/testbed clone --platform iOS --framework --app --app app-testbed -You will need to modify the ``iOS/testbed`` reference to point to that -directory in the CPython source tree; any folders specified with the ``--app`` -flag will be copied into the cloned testbed project. The resulting testbed will -be created in the ``app-testbed`` folder. In this example, the ``module1`` and -``module2`` would be importable modules at runtime. If your project has -additional dependencies, they can be installed into the -``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target -app-testbed/iOSTestbed/app_packages`` or similar). +Any folders specified with the ``--app`` flag will be copied into the cloned +testbed project. The resulting testbed will be created in the ``app-testbed`` +folder. In this example, the ``module1`` and ``module2`` would be importable +modules at runtime. If your project has additional dependencies, they can be +installed into the ``app-testbed/Testbed/app_packages`` folder (using ``pip +install --target app-testbed/Testbed/app_packages`` or similar). You can then use the ``app-testbed`` folder to run the test suite for your app, For example, if ``module1.tests`` was the entry point to your test suite, you @@ -381,7 +318,7 @@ tab. Modify the "Arguments Passed On Launch" value to change the testing arguments. The test plan also disables parallel testing, and specifies the use of the -``iOSTestbed.lldbinit`` file for providing configuration of the debugger. The +``Testbed.lldbinit`` file for providing configuration of the debugger. The default debugger configuration disables automatic breakpoints on the ``SIGINT``, ``SIGUSR1``, ``SIGUSR2``, and ``SIGXFSZ`` signals. diff --git a/Makefile.pre.in b/Makefile.pre.in index 764eef5be3ed2b..ba039794c88b01 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2315,7 +2315,7 @@ testios: fi # Clone the testbed project into the XCFOLDER - $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" + $(PYTHON_FOR_BUILD) $(srcdir)/Apple/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" # Run the testbed project $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run --verbose -- test -uall --single-process --rerun -W @@ -3219,10 +3219,10 @@ clean-retain-profile: pycremoval -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp - -rm -rf iOS/testbed/Python.xcframework/ios-*/bin - -rm -rf iOS/testbed/Python.xcframework/ios-*/lib - -rm -rf iOS/testbed/Python.xcframework/ios-*/include - -rm -rf iOS/testbed/Python.xcframework/ios-*/Python.framework + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/bin + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/lib + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/include + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/Python.framework .PHONY: profile-removal profile-removal: @@ -3248,7 +3248,7 @@ clobber: clean config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) - -rm -rf iOS/Frameworks + -rm -rf Apple/iOS/Frameworks -rm -rf iOSTestbed.* -rm -f python-config.py python-config -rm -rf cross-build diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-08-27-11-14-53.gh-issue-138171.Suz8ob.rst b/Misc/NEWS.d/next/Tools-Demos/2025-08-27-11-14-53.gh-issue-138171.Suz8ob.rst new file mode 100644 index 00000000000000..0a933ec754cdac --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-08-27-11-14-53.gh-issue-138171.Suz8ob.rst @@ -0,0 +1,3 @@ +A script for building an iOS XCframework was added. As part of this change, +the top level ``iOS`` folder has been moved to be a subdirectory of the +``Apple`` folder. diff --git a/configure b/configure index 6383271b477fd1..d31c24dffa2e9d 100755 --- a/configure +++ b/configure @@ -4359,7 +4359,7 @@ then : yes) case $ac_sys_system in Darwin) enableval=/Library/Frameworks ;; - iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + iOS) enableval=Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; *) as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 esac esac @@ -4470,9 +4470,9 @@ then : prefix=$PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" - RESSRCDIR=iOS/Resources + RESSRCDIR=Apple/iOS/Resources - ac_config_files="$ac_config_files iOS/Resources/Info.plist" + ac_config_files="$ac_config_files Apple/iOS/Resources/Info.plist" ;; *) @@ -35303,7 +35303,7 @@ do "Mac/PythonLauncher/Makefile") CONFIG_FILES="$CONFIG_FILES Mac/PythonLauncher/Makefile" ;; "Mac/Resources/framework/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/framework/Info.plist" ;; "Mac/Resources/app/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/app/Info.plist" ;; - "iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES iOS/Resources/Info.plist" ;; + "Apple/iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES Apple/iOS/Resources/Info.plist" ;; "Makefile.pre") CONFIG_FILES="$CONFIG_FILES Makefile.pre" ;; "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;; diff --git a/configure.ac b/configure.ac index 42d94776cc7ced..af7a9623d7bd79 100644 --- a/configure.ac +++ b/configure.ac @@ -559,7 +559,7 @@ AC_ARG_ENABLE([framework], yes) case $ac_sys_system in Darwin) enableval=/Library/Frameworks ;; - iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + iOS) enableval=Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; *) AC_MSG_ERROR([Unknown platform for framework build]) esac esac @@ -666,9 +666,9 @@ AC_ARG_ENABLE([framework], prefix=$PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" - RESSRCDIR=iOS/Resources + RESSRCDIR=Apple/iOS/Resources - AC_CONFIG_FILES([iOS/Resources/Info.plist]) + AC_CONFIG_FILES([Apple/iOS/Resources/Info.plist]) ;; *) AC_MSG_ERROR([Unknown platform for framework build]) diff --git a/iOS/README.rst b/iOS/README.rst deleted file mode 100644 index 4d38e5d7c307d1..00000000000000 --- a/iOS/README.rst +++ /dev/null @@ -1,352 +0,0 @@ -==================== -Python on iOS README -==================== - -:Authors: - Russell Keith-Magee (2023-11) - -This document provides a quick overview of some iOS specific features in the -Python distribution. - -These instructions are only needed if you're planning to compile Python for iOS -yourself. Most users should *not* need to do this. If you're looking to -experiment with writing an iOS app in Python, tools such as `BeeWare's Briefcase -`__ and `Kivy's Buildozer -`__ will provide a much more approachable -user experience. - -Compilers for building on iOS -============================= - -Building for iOS requires the use of Apple's Xcode tooling. It is strongly -recommended that you use the most recent stable release of Xcode. This will -require the use of the most (or second-most) recently released macOS version, -as Apple does not maintain Xcode for older macOS versions. The Xcode Command -Line Tools are not sufficient for iOS development; you need a *full* Xcode -install. - -If you want to run your code on the iOS simulator, you'll also need to install -an iOS Simulator Platform. You should be prompted to select an iOS Simulator -Platform when you first run Xcode. Alternatively, you can add an iOS Simulator -Platform by selecting an open the Platforms tab of the Xcode Settings panel. - -iOS specific arguments to configure -=================================== - -* ``--enable-framework[=DIR]`` - - This argument specifies the location where the Python.framework will be - installed. If ``DIR`` is not specified, the framework will be installed into - a subdirectory of the ``iOS/Frameworks`` folder. - - This argument *must* be provided when configuring iOS builds. iOS does not - support non-framework builds. - -* ``--with-framework-name=NAME`` - - Specify the name for the Python framework; defaults to ``Python``. - - .. admonition:: Use this option with care! - - Unless you know what you're doing, changing the name of the Python - framework on iOS is not advised. If you use this option, you won't be able - to run the ``make testios`` target without making significant manual - alterations, and you won't be able to use any binary packages unless you - compile them yourself using your own framework name. - -Building Python on iOS -====================== - -ABIs and Architectures ----------------------- - -iOS apps can be deployed on physical devices, and on the iOS simulator. Although -the API used on these devices is identical, the ABI is different - you need to -link against different libraries for an iOS device build (``iphoneos``) or an -iOS simulator build (``iphonesimulator``). - -Apple uses the ``XCframework`` format to allow specifying a single dependency -that supports multiple ABIs. An ``XCframework`` is a wrapper around multiple -ABI-specific frameworks that share a common API. - -iOS can also support different CPU architectures within each ABI. At present, -there is only a single supported architecture on physical devices - ARM64. -However, the *simulator* supports 2 architectures - ARM64 (for running on Apple -Silicon machines), and x86_64 (for running on older Intel-based machines). - -To support multiple CPU architectures on a single platform, Apple uses a "fat -binary" format - a single physical file that contains support for multiple -architectures. It is possible to compile and use a "thin" single architecture -version of a binary for testing purposes; however, the "thin" binary will not be -portable to machines using other architectures. - -Building a single-architecture framework ----------------------------------------- - -The Python build system will create a ``Python.framework`` that supports a -*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a -framework to contain non-library content, so the iOS build will produce a -``bin`` and ``lib`` folder in the same output folder as ``Python.framework``. -The ``lib`` folder will be needed at runtime to support the Python library. - -If you want to use Python in a real iOS project, you need to produce multiple -``Python.framework`` builds, one for each ABI and architecture. iOS builds of -Python *must* be constructed as framework builds. To support this, you must -provide the ``--enable-framework`` flag when configuring the build. The build -also requires the use of cross-compilation. The minimal commands for building -Python for the ARM64 iOS simulator will look something like:: - - $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" - $ ./configure \ - --enable-framework \ - --host=arm64-apple-ios-simulator \ - --build=arm64-apple-darwin \ - --with-build-python=/path/to/python.exe - $ make - $ make install - -In this invocation: - -* ``iOS/Resources/bin`` has been added to the path, providing some shims for the - compilers and linkers needed by the build. Xcode requires the use of ``xcrun`` - to invoke compiler tooling. However, if ``xcrun`` is pre-evaluated and the - result passed to ``configure``, these results can embed user- and - version-specific paths into the sysconfig data, which limits the portability - of the compiled Python. Alternatively, if ``xcrun`` is used *as* the compiler, - it requires that compiler variables like ``CC`` include spaces, which can - cause significant problems with many C configuration systems which assume that - ``CC`` will be a single executable. - - To work around this problem, the ``iOS/Resources/bin`` folder contains some - wrapper scripts that present as simple compilers and linkers, but wrap - underlying calls to ``xcrun``. This allows configure to use a ``CC`` - definition without spaces, and without user- or version-specific paths, while - retaining the ability to adapt to the local Xcode install. These scripts are - included in the ``bin`` directory of an iOS install. - - These scripts will, by default, use the currently active Xcode installation. - If you want to use a different Xcode installation, you can use - ``xcode-select`` to set a new default Xcode globally, or you can use the - ``DEVELOPER_DIR`` environment variable to specify an Xcode install. The - scripts will use the default ``iphoneos``/``iphonesimulator`` SDK version for - the select Xcode install; if you want to use a different SDK, you can set the - ``IOS_SDK_VERSION`` environment variable. (e.g, setting - ``IOS_SDK_VERSION=17.1`` would cause the scripts to use the ``iphoneos17.1`` - and ``iphonesimulator17.1`` SDKs, regardless of the Xcode default.) - - The path has also been cleared of any user customizations. A common source of - bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS - build. Resetting the path to a known "bare bones" value is the easiest way to - avoid these problems. - -* ``--host`` is the architecture and ABI that you want to build, in GNU compiler - triple format. This will be one of: - - - ``arm64-apple-ios`` for ARM64 iOS devices. - - ``arm64-apple-ios-simulator`` for the iOS simulator running on Apple - Silicon devices. - - ``x86_64-apple-ios-simulator`` for the iOS simulator running on Intel - devices. - -* ``--build`` is the GNU compiler triple for the machine that will be running - the compiler. This is one of: - - - ``arm64-apple-darwin`` for Apple Silicon devices. - - ``x86_64-apple-darwin`` for Intel devices. - -* ``/path/to/python.exe`` is the path to a Python binary on the machine that - will be running the compiler. This is needed because the Python compilation - process involves running some Python code. On a normal desktop build of - Python, you can compile a python interpreter and then use that interpreter to - run Python code. However, the binaries produced for iOS won't run on macOS, so - you need to provide an external Python interpreter. This interpreter must be - the same version as the Python that is being compiled. To be completely safe, - this should be the *exact* same commit hash. However, the longer a Python - release has been stable, the more likely it is that this constraint can be - relaxed - the same micro version will often be sufficient. - -* The ``install`` target for iOS builds is slightly different to other - platforms. On most platforms, ``make install`` will install the build into - the final runtime location. This won't be the case for iOS, as the final - runtime location will be on a physical device. - - However, you still need to run the ``install`` target for iOS builds, as it - performs some final framework assembly steps. The location specified with - ``--enable-framework`` will be the location where ``make install`` will - assemble the complete iOS framework. This completed framework can then - be copied and relocated as required. - -For a full CPython build, you also need to specify the paths to iOS builds of -the binary libraries that CPython depends on (XZ, BZip2, LibFFI and OpenSSL). -This can be done by defining the ``LIBLZMA_CFLAGS``, ``LIBLZMA_LIBS``, -``BZIP2_CFLAGS``, ``BZIP2_LIBS``, ``LIBFFI_CFLAGS``, and ``LIBFFI_LIBS`` -environment variables, and the ``--with-openssl`` configure option. Versions of -these libraries pre-compiled for iOS can be found in `this repository -`__. LibFFI is -especially important, as many parts of the standard library (including the -``platform``, ``sysconfig`` and ``webbrowser`` modules) require the use of the -``ctypes`` module at runtime. - -By default, Python will be compiled with an iOS deployment target (i.e., the -minimum supported iOS version) of 13.0. To specify a different deployment -target, provide the version number as part of the ``--host`` argument - for -example, ``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 -simulator build with a deployment target of 15.4. - -Merge thin frameworks into fat frameworks ------------------------------------------ - -Once you've built a ``Python.framework`` for each ABI and architecture, you -must produce a "fat" framework for each ABI that contains all the architectures -for that ABI. - -The ``iphoneos`` build only needs to support a single architecture, so it can be -used without modification. - -If you only want to support a single simulator architecture, (e.g., only support -ARM64 simulators), you can use a single architecture ``Python.framework`` build. -However, if you want to create ``Python.xcframework`` that supports *all* -architectures, you'll need to merge the ``iphonesimulator`` builds for ARM64 and -x86_64 into a single "fat" framework. - -The "fat" framework can be constructed by performing a directory merge of the -content of the two "thin" ``Python.framework`` directories, plus the ``bin`` and -``lib`` folders for each thin framework. When performing this merge: - -* The pure Python standard library content is identical for each architecture, - except for a handful of platform-specific files (such as the ``sysconfig`` - module). Ensure that the "fat" framework has the union of all standard library - files. - -* Any binary files in the standard library, plus the main - ``libPython3.X.dylib``, can be merged using the ``lipo`` tool, provide by - Xcode:: - - $ lipo -create -output module.dylib path/to/x86_64/module.dylib path/to/arm64/module.dylib - -* The header files will be identical on both architectures, except for - ``pyconfig.h``. Copy all the headers from one platform (say, arm64), rename - ``pyconfig.h`` to ``pyconfig-arm64.h``, and copy the ``pyconfig.h`` for the - other architecture into the merged header folder as ``pyconfig-x86_64.h``. - Then copy the ``iOS/Resources/pyconfig.h`` file from the CPython sources into - the merged headers folder. This will allow the two Python architectures to - share a common ``pyconfig.h`` header file. - -At this point, you should have 2 Python.framework folders - one for ``iphoneos``, -and one for ``iphonesimulator`` that is a merge of x86+64 and ARM64 content. - -Merge frameworks into an XCframework ------------------------------------- - -Now that we have 2 (potentially fat) ABI-specific frameworks, we can merge those -frameworks into a single ``XCframework``. - -The initial skeleton of an ``XCframework`` is built using:: - - xcodebuild -create-xcframework -output Python.xcframework -framework path/to/iphoneos/Python.framework -framework path/to/iphonesimulator/Python.framework - -Then, copy the ``bin`` and ``lib`` folders into the architecture-specific slices of -the XCframework:: - - cp path/to/iphoneos/bin Python.xcframework/ios-arm64 - cp path/to/iphoneos/lib Python.xcframework/ios-arm64 - - cp path/to/iphonesimulator/bin Python.xcframework/ios-arm64_x86_64-simulator - cp path/to/iphonesimulator/lib Python.xcframework/ios-arm64_x86_64-simulator - -Note that the name of the architecture-specific slice for the simulator will -depend on the CPU architecture(s) that you build. - -You now have a Python.xcframework that can be used in a project. - -Testing Python on iOS -===================== - -The ``iOS/testbed`` folder that contains an Xcode project that is able to run -the iOS test suite. This project converts the Python test suite into a single -test case in Xcode's XCTest framework. The single XCTest passes if the test -suite passes. - -To run the test suite, configure a Python build for an iOS simulator (i.e., -``--host=arm64-apple-ios-simulator`` or ``--host=x86_64-apple-ios-simulator`` -), specifying a framework build (i.e. ``--enable-framework``). Ensure that your -``PATH`` has been configured to include the ``iOS/Resources/bin`` folder and -exclude any non-iOS tools, then run:: - - $ make all - $ make install - $ make testios - -This will: - -* Build an iOS framework for your chosen architecture; -* Finalize the single-platform framework; -* Make a clean copy of the testbed project; -* Install the Python iOS framework into the copy of the testbed project; and -* Run the test suite on an "iPhone SE (3rd generation)" simulator. - -On success, the test suite will exit and report successful completion of the -test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 -minutes to run; a couple of extra minutes is required to compile the testbed -project, and then boot and prepare the iOS simulator. - -Debugging test failures ------------------------ - -Running ``make testios`` generates a standalone version of the ``iOS/testbed`` -project, and runs the full test suite. It does this using ``iOS/testbed`` -itself - the folder is an executable module that can be used to create and run -a clone of the testbed project. - -You can generate your own standalone testbed instance by running:: - - $ python iOS/testbed clone --framework iOS/Frameworks/arm64-iphonesimulator my-testbed - -This invocation assumes that ``iOS/Frameworks/arm64-iphonesimulator`` is the -path to the iOS simulator framework for your platform (ARM64 in this case); -``my-testbed`` is the name of the folder for the new testbed clone. - -You can then use the ``my-testbed`` folder to run the Python test suite, -passing in any command line arguments you may require. For example, if you're -trying to diagnose a failure in the ``os`` module, you might run:: - - $ python my-testbed run -- test -W test_os - -This is the equivalent of running ``python -m test -W test_os`` on a desktop -Python build. Any arguments after the ``--`` will be passed to testbed as if -they were arguments to ``python -m`` on a desktop machine. - -Testing in Xcode -^^^^^^^^^^^^^^^^ - -You can also open the testbed project in Xcode by running:: - - $ open my-testbed/iOSTestbed.xcodeproj - -This will allow you to use the full Xcode suite of tools for debugging. - -The arguments used to run the test suite are defined as part of the test plan. -To modify the test plan, select the test plan node of the project tree (it -should be the first child of the root node), and select the "Configurations" -tab. Modify the "Arguments Passed On Launch" value to change the testing -arguments. - -The test plan also disables parallel testing, and specifies the use of the -``iOSTestbed.lldbinit`` file for providing configuration of the debugger. The -default debugger configuration disables automatic breakpoints on the -``SIGINT``, ``SIGUSR1``, ``SIGUSR2``, and ``SIGXFSZ`` signals. - -Testing on an iOS device -^^^^^^^^^^^^^^^^^^^^^^^^ - -To test on an iOS device, the app needs to be signed with known developer -credentials. To obtain these credentials, you must have an iOS Developer -account, and your Xcode install will need to be logged into your account (see -the Accounts tab of the Preferences dialog). - -Once the project is open, and you're signed into your Apple Developer account, -select the root node of the project tree (labeled "iOSTestbed"), then the -"Signing & Capabilities" tab in the details page. Select a development team -(this will likely be your own name), and plug in a physical device to your -macOS machine with a USB cable. You should then be able to select your physical -device from the list of targets in the pulldown in the Xcode titlebar. diff --git a/iOS/Resources/dylib-Info-template.plist b/iOS/Resources/dylib-Info-template.plist deleted file mode 100644 index f652e272f71c88..00000000000000 --- a/iOS/Resources/dylib-Info-template.plist +++ /dev/null @@ -1,26 +0,0 @@ - - - - - CFBundleDevelopmentRegion - en - CFBundleExecutable - - CFBundleIdentifier - - CFBundleInfoDictionaryVersion - 6.0 - CFBundlePackageType - APPL - CFBundleShortVersionString - 1.0 - CFBundleSupportedPlatforms - - iPhoneOS - - MinimumOSVersion - 12.0 - CFBundleVersion - 1 - -