From f53f59946816280486dd54b282a711a0b8fce697 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 8 Dec 2024 12:50:41 -0500 Subject: [PATCH 1/4] Use a non-versioned Xcode path in sysconfig --- .github/workflows/apple.yml | 152 ---------------------------------- .github/workflows/linux.yml | 1 - .github/workflows/windows.yml | 1 - cpython-unix/build-cpython.sh | 4 + cpython-unix/build.py | 23 ++--- 5 files changed, 10 insertions(+), 171 deletions(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index f20bc3c4..4059d6d8 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -45,163 +45,11 @@ jobs: fail-fast: false matrix: build: - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.9' - options: 'pgo' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.9' - options: 'pgo+lto' - - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.10' - options: 'pgo' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.10' - options: 'pgo+lto' - - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.11' - options: 'pgo' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.11' - options: 'pgo+lto' - - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.12' options: 'debug' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.12' - options: 'pgo' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.12' - options: 'pgo+lto' - - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.13' - options: 'pgo' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.13' - options: 'pgo+lto' - - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.13' - options: 'freethreaded+debug' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.13' - options: 'freethreaded+pgo' - - target_triple: 'aarch64-apple-darwin' - runner: macos-14 - py: 'cpython-3.13' - options: 'freethreaded+pgo+lto' - - # macOS on Intel hardware. This is pretty straightforward. We exclude - # noopt because it doesn't provide any compelling advantages over PGO - # or LTO builds. - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.9' - options: 'debug' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.9' - options: 'pgo' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.9' - options: 'pgo+lto' - - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.10' - options: 'debug' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.10' - options: 'pgo' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.10' - options: 'pgo+lto' - - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.11' - options: 'debug' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.11' - options: 'pgo' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.11' - options: 'pgo+lto' - - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.12' - options: 'debug' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.12' - options: 'pgo' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.12' - options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.13' - options: 'debug' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.13' - options: 'pgo' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.13' - options: 'pgo+lto' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.13' - options: 'freethreaded+debug' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.13' - options: 'freethreaded+pgo' - - target_triple: 'x86_64-apple-darwin' - runner: macos-13 - py: 'cpython-3.13' - options: 'freethreaded+pgo+lto' needs: - pythonbuild runs-on: ${{ matrix.build.runner }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index c64628e8..8b15d06d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -3,7 +3,6 @@ name: Linux Python build on: push: branches: [main] - pull_request: concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 18b83895..e04c6790 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -3,7 +3,6 @@ name: Windows Python build on: push: branches: [main] - pull_request: concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 7c969d30..f61945e6 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -627,6 +627,10 @@ def replace_in_all(search, replace): replace_in_file(MAKEFILE, search, replace) replace_in_file(SYSCONFIGDATA, search, replace) +# Replace the XCode path with a generic value. +xcode_path = os.getenv("APPLE_SDK_PATH") +if xcode_path: + replace_in_all(xcode_path, "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") # -fdebug-default-version is Clang only. Strip so compiling works on GCC. replace_in_all("-fdebug-default-version=4", "") diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 1c1963bb..4794ea14 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -165,23 +165,7 @@ def add_target_env(env, build_platform, target_triple, build_env): # non-system (e.g. Homebrew) executables from being used. env["PATH"] = "/usr/bin:/bin" - if "APPLE_SDK_PATH" in os.environ: - sdk_path = os.environ["APPLE_SDK_PATH"] - else: - # macOS SDK has historically been in /usr courtesy of an - # installer provided by Xcode. But with Catalina, the files - # are now typically in - # /Applications/Xcode.app/Contents/Developer/Platforms/. - # The proper way to resolve this path is with xcrun, which - # will give us the headers that Xcode is configured to use. - res = subprocess.run( - ["xcrun", "--sdk", sdk_platform, "--show-sdk-path"], - check=True, - capture_output=True, - encoding="utf-8", - ) - - sdk_path = res.stdout.strip() + sdk_path = os.environ["APPLE_SDK_PATH"] if not os.path.exists(sdk_path): raise Exception("macOS SDK path %s does not exist" % sdk_path) @@ -817,6 +801,11 @@ def build_cpython( if "lto" in parsed_build_options: env["CPYTHON_LTO"] = "1" + sdk_path = os.environ["APPLE_SDK_PATH"] + if not os.path.exists(sdk_path): + raise Exception("macOS SDK path %s does not exist" % sdk_path) + env["APPLE_SDK_PATH"] = sdk_path + add_target_env(env, host_platform, target_triple, build_env) build_env.run("build-cpython.sh", environment=env) From eaa75563ec850c344aa95848b7e5df2b92722305 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 8 Dec 2024 14:43:39 -0500 Subject: [PATCH 2/4] Drop from _sysconfig --- .github/workflows/apple.yml | 5 ++--- cpython-unix/build-cpython.sh | 8 ++++++-- cpython-unix/build.py | 25 +++++++++++++++++++------ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 4059d6d8..24dfdf0b 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -47,9 +47,8 @@ jobs: build: - target_triple: 'aarch64-apple-darwin' runner: macos-14 - py: 'cpython-3.12' - options: 'debug' - + py: 'cpython-3.13' + options: 'freethreaded+debug' needs: - pythonbuild runs-on: ${{ matrix.build.runner }} diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index f61945e6..b6c3caec 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -627,10 +627,14 @@ def replace_in_all(search, replace): replace_in_file(MAKEFILE, search, replace) replace_in_file(SYSCONFIGDATA, search, replace) -# Replace the XCode path with a generic value. +# Remove the Xcode path from the compiler flags. +# +# CPython itself will drop this from `sysconfig.get_config_var("CFLAGS")` and +# similar calls, but _not_ if `CFLAGS` is set in the environment (regardless of +# the `CFLAGS` value). It will almost always be wrong, so we drop it unconditionally. xcode_path = os.getenv("APPLE_SDK_PATH") if xcode_path: - replace_in_all(xcode_path, "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") + replace_in_all("-isysroot %s" % xcode_path, "") # -fdebug-default-version is Clang only. Strip so compiling works on GCC. replace_in_all("-fdebug-default-version=4", "") diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 4794ea14..9bfc93ef 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -165,11 +165,29 @@ def add_target_env(env, build_platform, target_triple, build_env): # non-system (e.g. Homebrew) executables from being used. env["PATH"] = "/usr/bin:/bin" - sdk_path = os.environ["APPLE_SDK_PATH"] + if "APPLE_SDK_PATH" in os.environ: + sdk_path = os.environ["APPLE_SDK_PATH"] + else: + # macOS SDK has historically been in /usr courtesy of an + # installer provided by Xcode. But with Catalina, the files + # are now typically in + # /Applications/Xcode.app/Contents/Developer/Platforms/. + # The proper way to resolve this path is with xcrun, which + # will give us the headers that Xcode is configured to use. + res = subprocess.run( + ["xcrun", "--sdk", sdk_platform, "--show-sdk-path"], + check=True, + capture_output=True, + encoding="utf-8", + ) + + sdk_path = res.stdout.strip() if not os.path.exists(sdk_path): raise Exception("macOS SDK path %s does not exist" % sdk_path) + env["APPLE_SDK_PATH"] = sdk_path + # Grab the version from the SDK so we can put it in PYTHON.json. sdk_settings_path = pathlib.Path(sdk_path) / "SDKSettings.json" with sdk_settings_path.open("rb") as fh: @@ -801,11 +819,6 @@ def build_cpython( if "lto" in parsed_build_options: env["CPYTHON_LTO"] = "1" - sdk_path = os.environ["APPLE_SDK_PATH"] - if not os.path.exists(sdk_path): - raise Exception("macOS SDK path %s does not exist" % sdk_path) - env["APPLE_SDK_PATH"] = sdk_path - add_target_env(env, host_platform, target_triple, build_env) build_env.run("build-cpython.sh", environment=env) From 8d404053347abd23c5bc1a5f9cc2004389628a9b Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 8 Dec 2024 17:37:49 -0500 Subject: [PATCH 3/4] Reformat sysconfig --- .github/workflows/apple.yml | 12 +++++++++++- cpython-unix/build-cpython.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 24dfdf0b..6e41e41a 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -45,10 +45,20 @@ jobs: fail-fast: false matrix: build: + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.9' + options: 'debug' + + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.12' + options: 'debug' + - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.13' - options: 'freethreaded+debug' + options: 'debug' needs: - pythonbuild runs-on: ${{ matrix.build.runner }} diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index b6c3caec..23ed5dca 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -587,6 +587,7 @@ fi # that a) it works on as many machines as possible b) doesn't leak details # about the build environment, which is non-portable. cat > ${ROOT}/hack_sysconfig.py << EOF +import json import os import sys import sysconfig @@ -627,6 +628,33 @@ def replace_in_all(search, replace): replace_in_file(MAKEFILE, search, replace) replace_in_file(SYSCONFIGDATA, search, replace) + +def format_sysconfigdata(): + """Reformat the sysconfigdata file to avoid implicit string concatenations. + + In some Python versions, the sysconfigdata file contains implicit string + concatenations that extend over multiple lines, which make string replacement + much harder. This function reformats the file to avoid this issue. + + See: https://github.com/python/cpython/blob/a03efb533a58fd13fb0cc7f4a5c02c8406a407bd/Mac/BuildScript/build-installer.py#L1360C1-L1385C15. + """ + with open(SYSCONFIGDATA, "rb") as fh: + data = fh.read() + + globals_dict = {} + locals_dict = {} + exec(data, globals_dict, locals_dict) + build_time_vars = locals_dict['build_time_vars'] + + with open(SYSCONFIGDATA, "wb") as fh: + fh.write(b'# system configuration generated and used by the sysconfig module\n') + fh.write(('build_time_vars = %s' % json.dumps(build_time_vars, indent=4)).encode("utf-8")) + fh.close() + + +# Format sysconfig to ensure that string replacements take effect. +format_sysconfigdata() + # Remove the Xcode path from the compiler flags. # # CPython itself will drop this from `sysconfig.get_config_var("CFLAGS")` and From 6ed2b07fd6b823a4e5d6872bcb1b0a18ecc29d53 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 8 Dec 2024 18:30:07 -0500 Subject: [PATCH 4/4] Reverts --- .github/workflows/apple.yml | 143 ++++++++++++++++++++++++++++++++++ .github/workflows/linux.yml | 1 + .github/workflows/windows.yml | 1 + 3 files changed, 145 insertions(+) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 6e41e41a..f20bc3c4 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -49,16 +49,159 @@ jobs: runner: macos-14 py: 'cpython-3.9' options: 'debug' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.9' + options: 'pgo' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.9' + options: 'pgo+lto' + + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.10' + options: 'debug' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.10' + options: 'pgo' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.10' + options: 'pgo+lto' + + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.11' + options: 'debug' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.11' + options: 'pgo' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.11' + options: 'pgo+lto' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.12' options: 'debug' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.12' + options: 'pgo' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.12' + options: 'pgo+lto' + + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.13' + options: 'debug' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.13' + options: 'pgo' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.13' + options: 'pgo+lto' - target_triple: 'aarch64-apple-darwin' runner: macos-14 py: 'cpython-3.13' + options: 'freethreaded+debug' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.13' + options: 'freethreaded+pgo' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.13' + options: 'freethreaded+pgo+lto' + + # macOS on Intel hardware. This is pretty straightforward. We exclude + # noopt because it doesn't provide any compelling advantages over PGO + # or LTO builds. + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.9' + options: 'debug' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.9' + options: 'pgo' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.9' + options: 'pgo+lto' + + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.10' + options: 'debug' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.10' + options: 'pgo' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.10' + options: 'pgo+lto' + + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.11' + options: 'debug' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.11' + options: 'pgo' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.11' + options: 'pgo+lto' + + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.12' + options: 'debug' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.12' + options: 'pgo' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.12' + options: 'pgo+lto' + + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.13' options: 'debug' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.13' + options: 'pgo' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.13' + options: 'pgo+lto' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.13' + options: 'freethreaded+debug' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.13' + options: 'freethreaded+pgo' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.13' + options: 'freethreaded+pgo+lto' needs: - pythonbuild runs-on: ${{ matrix.build.runner }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 8b15d06d..c64628e8 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -3,6 +3,7 @@ name: Linux Python build on: push: branches: [main] + pull_request: concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e04c6790..18b83895 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -3,6 +3,7 @@ name: Windows Python build on: push: branches: [main] + pull_request: concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}