From 18a1a06e24dacde74cd48b98fdda9ed23245502d Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 30 Jul 2025 15:59:49 +0100 Subject: [PATCH 01/20] Add Android release workflow --- ...and-docs-release.yml => build-release.yml} | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) rename .github/workflows/{source-and-docs-release.yml => build-release.yml} (82%) diff --git a/.github/workflows/source-and-docs-release.yml b/.github/workflows/build-release.yml similarity index 82% rename from .github/workflows/source-and-docs-release.yml rename to .github/workflows/build-release.yml index 6cb2e5fc..dc0541c7 100644 --- a/.github/workflows/source-and-docs-release.yml +++ b/.github/workflows/build-release.yml @@ -35,7 +35,7 @@ on: type: string description: "CPython release number (ie '3.11.5', note without the 'v' prefix)" -name: "Build Python source and docs artifacts" +name: "Build Python release artifacts" permissions: {} @@ -184,3 +184,39 @@ jobs: cd ../installation ./bin/python3 -m test -uall + + build-android: + needs: + - verify-input + + # Android binary releases began in Python 3.14. + if: | + !( + startsWith(needs.verify-input.outputs.cpython_release, '3.9.') || + startsWith(needs.verify-input.outputs.cpython_release, '3.10.') || + startsWith(needs.verify-input.outputs.cpython_release, '3.11.') || + startsWith(needs.verify-input.outputs.cpython_release, '3.12.') || + startsWith(needs.verify-input.outputs.cpython_release, '3.13.') + ) + timeout-minutes: 60 + strategy: + matrix: + arch: [aarch64, x86_64] + include: + - arch: aarch64 + runs-on: macos-15 + - arch: x86_64 + runs-on: ubuntu-24.04 + + runs-on: ${{ matrix.runs-on }} + steps: + - name: "Checkout ${{ env.GIT_REMOTE }}/cpython" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + repository: "${{ env.GIT_REMOTE }}/cpython" + ref: "v${{ env.CPYTHON_RELEASE }}" + + - uses: ./.github/actions/build-android + with: + triplet: ${{ matrix.arch }}-linux-android From 081e3beac0fb626798bea049d3be9caf32fd2986 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 30 Jul 2025 16:29:01 +0100 Subject: [PATCH 02/20] Temporarily point inputs at https://github.com/python/cpython/pull/137186 --- .github/workflows/build-release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index dc0541c7..b6fd60a0 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -41,9 +41,9 @@ permissions: {} # Set from inputs for workflow_dispatch, or set defaults to test push/PR events env: - GIT_REMOTE: ${{ github.event.inputs.git_remote || 'python' }} - GIT_COMMIT: ${{ github.event.inputs.git_commit || '4f8bb3947cfbc20f970ff9d9531e1132a9e95396' }} - CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.13.2' }} + GIT_REMOTE: ${{ github.event.inputs.git_remote || 'mhsmith' }} + GIT_COMMIT: ${{ github.event.inputs.git_commit || '16f1230f241d69353cb75cb5856bffd3fc0de9f4' }} + CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.15.991' }} jobs: verify-input: From a698e15158f07e585256624a1867b67f23dd3199 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 30 Jul 2025 17:06:35 +0100 Subject: [PATCH 03/20] Revert to previous inputs --- .github/workflows/build-release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index b6fd60a0..dc0541c7 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -41,9 +41,9 @@ permissions: {} # Set from inputs for workflow_dispatch, or set defaults to test push/PR events env: - GIT_REMOTE: ${{ github.event.inputs.git_remote || 'mhsmith' }} - GIT_COMMIT: ${{ github.event.inputs.git_commit || '16f1230f241d69353cb75cb5856bffd3fc0de9f4' }} - CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.15.991' }} + GIT_REMOTE: ${{ github.event.inputs.git_remote || 'python' }} + GIT_COMMIT: ${{ github.event.inputs.git_commit || '4f8bb3947cfbc20f970ff9d9531e1132a9e95396' }} + CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.13.2' }} jobs: verify-input: From fa37a7f7bbcf37c62de4c8c7a2672db7362e5ca4 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 4 Aug 2025 14:41:53 +0100 Subject: [PATCH 04/20] Update references to workflow name --- run_release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_release.py b/run_release.py index 891290cf..d15ab031 100755 --- a/run_release.py +++ b/run_release.py @@ -906,7 +906,7 @@ def start_build_of_source_and_docs(db: ReleaseShelf) -> None: # with the known good commit SHA. print() print( - "Go to https://github.com/python/release-tools/actions/workflows/source-and-docs-release.yml" + "Go to https://github.com/python/release-tools/actions/workflows/build-release.yml" ) print("Select 'Run workflow' and enter the following values:") print(f"- Git remote to checkout: {origin_remote_github_owner}") @@ -915,7 +915,7 @@ def start_build_of_source_and_docs(db: ReleaseShelf) -> None: print() print("Or using the GitHub CLI run:") print( - " gh workflow run source-and-docs-release.yml --repo python/release-tools" + " gh workflow run build-release.yml --repo python/release-tools" f" -f git_remote={origin_remote_github_owner}" f" -f git_commit={commit_sha}" f" -f cpython_release={db['release']}" From cc0dbdda39a80572ca715bb12f73568cb866a909 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 6 Aug 2025 23:15:28 +0100 Subject: [PATCH 05/20] Add Android to add_to_pydotorg.py --- add_to_pydotorg.py | 141 ++++++++++++++++++++++------------ tests/fake-ftp-files.txt | 2 + tests/test_add_to_pydotorg.py | 39 +++++++--- 3 files changed, 121 insertions(+), 61 deletions(-) diff --git a/add_to_pydotorg.py b/add_to_pydotorg.py index a225189f..fc5af85c 100755 --- a/add_to_pydotorg.py +++ b/add_to_pydotorg.py @@ -4,11 +4,10 @@ To use (RELEASE is something like 3.3.5rc2): * Copy this script to dl-files (it needs access to all the release files). - You could also download all files, then you need to adapt the "ftp_root" - string below. + You could also download all files, then you need to use the "--ftp-root" + argument. -* Make sure all download files are in place in the correct /srv/www.python.org - subdirectory. +* Make sure all download files are in place in the correct FTP subdirectory. * Create a new Release object via the Django admin (adding via API is currently broken), the name MUST be "Python RELEASE". @@ -23,6 +22,7 @@ Georg Brandl, March 2014. """ +import argparse import hashlib import json import os @@ -70,8 +70,6 @@ def run_cmd( ) sys.exit() -base_url = "https://www.python.org/api/v1/" -ftp_root = "/srv/www.python.org/ftp/python/" download_root = "https://www.python.org/ftp/python/" tag_cre = re.compile(r"(\d+)(?:\.(\d+)(?:\.(\d+))?)?(?:([ab]|rc)(\d+))?$") @@ -98,44 +96,47 @@ def get_file_descriptions( ) -> list[tuple[re.Pattern[str], tuple[str, int, bool, str]]]: v = minor_version_tuple(release) rx = re.compile - # value is (file "name", OS id, download button, file "description"). - # OS=0 means no ReleaseFile object. Only one matching *file* (not regex) + # value is (file "name", OS slug, download button, file "description"). + # OS=None means no ReleaseFile object. Only one matching *file* (not regex) # per OS can have download=True. return [ - (rx(r"\.tgz$"), ("Gzipped source tarball", 3, False, "")), - (rx(r"\.tar\.xz$"), ("XZ compressed source tarball", 3, True, "")), + (rx(r"\.tgz$"), ("Gzipped source tarball", "source", False, "")), + (rx(r"\.tar\.xz$"), ("XZ compressed source tarball", "source", True, "")), ( rx(r"windows-.+\.json"), ( "Windows release manifest", - 1, + "windows", False, f"Install with 'py install {v[0]}.{v[1]}'", ), ), ( rx(r"-embed-amd64\.zip$"), - ("Windows embeddable package (64-bit)", 1, False, ""), + ("Windows embeddable package (64-bit)", "windows", False, ""), ), ( rx(r"-embed-arm64\.zip$"), - ("Windows embeddable package (ARM64)", 1, False, ""), + ("Windows embeddable package (ARM64)", "windows", False, ""), + ), + ( + rx(r"-arm64\.exe$"), + ("Windows installer (ARM64)", "windows", False, "Experimental"), ), - (rx(r"-arm64\.exe$"), ("Windows installer (ARM64)", 1, False, "Experimental")), ( rx(r"-amd64\.exe$"), - ("Windows installer (64-bit)", 1, v >= (3, 9), "Recommended"), + ("Windows installer (64-bit)", "windows", v >= (3, 9), "Recommended"), ), ( rx(r"-embed-win32\.zip$"), - ("Windows embeddable package (32-bit)", 1, False, ""), + ("Windows embeddable package (32-bit)", "windows", False, ""), ), - (rx(r"\.exe$"), ("Windows installer (32-bit)", 1, v < (3, 9), "")), + (rx(r"\.exe$"), ("Windows installer (32-bit)", "windows", v < (3, 9), "")), ( rx(r"-macosx10\.5(_rev\d)?\.(dm|pk)g$"), ( "macOS 32-bit i386/PPC installer", - 2, + "macos", False, "for Mac OS X 10.5 and later", ), @@ -144,7 +145,7 @@ def get_file_descriptions( rx(r"-macosx10\.6(_rev\d)?\.(dm|pk)g$"), ( "macOS 64-bit/32-bit Intel installer", - 2, + "macos", False, "for Mac OS X 10.6 and later", ), @@ -153,7 +154,7 @@ def get_file_descriptions( rx(r"-macos(x)?10\.9\.(dm|pk)g$"), ( "macOS 64-bit Intel-only installer", - 2, + "macos", False, "for macOS 10.9 and later, deprecated", ), @@ -162,11 +163,19 @@ def get_file_descriptions( rx(r"-macos(x)?1[1-9](\.[0-9]*)?\.pkg$"), ( "macOS 64-bit universal2 installer", - 2, + "macos", True, f"for macOS {'10.13' if v >= (3, 12, 6) else '10.9'} and later", ), ), + ( + rx(r"aarch64-linux-android.tar.gz$"), + ("Android embeddable package (aarch64)", "android", False, ""), + ), + ( + rx(r"x86_64-linux-android.tar.gz$"), + ("Android embeddable package (x86_64)", "android", False, ""), + ), ] @@ -182,14 +191,14 @@ def sigfile_for(release: str, rfile: str) -> str: return download_root + f"{release}/{rfile}.asc" -def md5sum_for(release: str, rfile: str) -> str: +def md5sum_for(filename: str) -> str: return hashlib.md5( - open(ftp_root + base_version(release) + "/" + rfile, "rb").read() + open(filename, "rb").read(), ).hexdigest() -def filesize_for(release: str, rfile: str) -> int: - return path.getsize(ftp_root + base_version(release) + "/" + rfile) +def filesize_for(filename: str) -> int: + return path.getsize(filename) def make_slug(text: str) -> str: @@ -215,6 +224,7 @@ def minor_version_tuple(release: str) -> tuple[int, int]: def build_file_dict( + ftp_root: str, release: str, rfile: str, rel_pk: int, @@ -224,6 +234,7 @@ def build_file_dict( add_desc: str, ) -> dict[str, Any]: """Return a dictionary with all needed fields for a ReleaseFile object.""" + filename = path.join(ftp_root, base_version(release), rfile) d = { "name": file_desc, "slug": slug_for(release) + "-" + make_slug(file_desc)[:40], @@ -232,28 +243,28 @@ def build_file_dict( "description": add_desc, "is_source": os_pk == 3, "url": download_root + f"{base_version(release)}/{rfile}", - "md5_sum": md5sum_for(release, rfile), - "filesize": filesize_for(release, rfile), + "md5_sum": md5sum_for(filename), + "filesize": filesize_for(filename), "download_button": add_download, } # Upload GPG signature - if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.asc"): + if os.path.exists(filename + ".asc"): d["gpg_signature_file"] = sigfile_for(base_version(release), rfile) # Upload Sigstore signature - if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sig"): + if os.path.exists(filename + ".sig"): d["sigstore_signature_file"] = ( download_root + f"{base_version(release)}/{rfile}.sig" ) # Upload Sigstore certificate - if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.crt"): + if os.path.exists(filename + ".crt"): d["sigstore_cert_file"] = download_root + f"{base_version(release)}/{rfile}.crt" # Upload Sigstore bundle - if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sigstore"): + if os.path.exists(filename + ".sigstore"): d["sigstore_bundle_file"] = ( download_root + f"{base_version(release)}/{rfile}.sigstore" ) # Upload SPDX SBOM file - if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.spdx.json"): + if os.path.exists(filename + ".spdx.json"): d["sbom_spdx2_file"] = ( download_root + f"{base_version(release)}/{rfile}.spdx.json" ) @@ -261,7 +272,9 @@ def build_file_dict( return d -def list_files(release: str) -> Generator[tuple[str, str, int, bool, str], None, None]: +def list_files( + ftp_root: str, release: str +) -> Generator[tuple[str, str, int, bool, str], None, None]: """List all of the release's download files.""" reldir = base_version(release) for rfile in os.listdir(path.join(ftp_root, reldir)): @@ -283,15 +296,14 @@ def list_files(release: str) -> Generator[tuple[str, str, int, bool, str], None, for rx, info in get_file_descriptions(release): if rx.search(rfile): - file_desc, os_pk, add_download, add_desc = info - yield rfile, file_desc, os_pk, add_download, add_desc + yield (rfile, *info) break else: print(f" File {reldir}/{rfile} not recognized") continue -def query_object(objtype: str, **params: Any) -> int: +def query_object(base_url: str, objtype: str, **params: Any) -> int: """Find an API object by query parameters.""" uri = base_url + f"downloads/{objtype}/" uri += "?" + "&".join(f"{k}={v}" for k, v in params.items()) @@ -302,7 +314,7 @@ def query_object(objtype: str, **params: Any) -> int: return int(obj["resource_uri"].strip("/").split("/")[-1]) -def post_object(objtype: str, datadict: dict[str, Any]) -> int: +def post_object(base_url: str, objtype: str, datadict: dict[str, Any]) -> int: """Create a new API object.""" resp = requests.post( base_url + "downloads/" + objtype + "/", @@ -324,11 +336,11 @@ def post_object(objtype: str, datadict: dict[str, Any]) -> int: def sign_release_files_with_sigstore( - release: str, release_files: list[tuple[str, str, int, bool, str]] + ftp_root: str, release: str, release_files: list[tuple[str, str, int, bool, str]] ) -> None: filenames = [ ftp_root + f"{base_version(release)}/{rfile}" - for rfile, file_desc, os_pk, add_download, add_desc in release_files + for rfile, *_ in release_files ] def has_sigstore_signature(filename: str) -> bool: @@ -445,34 +457,65 @@ def has_sigstore_signature(filename: str) -> bool: ) +def parse_args() -> argparse.Namespace: + def ensure_trailing_slash(s: str): + if not s.endswith("/"): + s += "/" + return s + + parser = argparse.ArgumentParser() + parser.add_argument( + "--base-url", + metavar="URL", + type=ensure_trailing_slash, + default="https://www.python.org/api/v1/", + help="API URL; defaults to %(default)s", + ) + parser.add_argument( + "--ftp-root", + metavar="DIR", + type=ensure_trailing_slash, + default="/srv/www.python.org/ftp/python", + help="FTP root; defaults to %(default)s", + ) + parser.add_argument( + "release", + help="Python version number, e.g. 3.3.5rc2", + ) + return parser.parse_args() + + def main() -> None: - rel = sys.argv[1] + args = parse_args() + rel = args.release print("Querying python.org for release", rel) - rel_pk = query_object("release", name="Python+" + rel) + rel_pk = query_object(args.base_url, "release", name="Python+" + rel) print("Found Release object: id =", rel_pk) - release_files = list(list_files(rel)) - sign_release_files_with_sigstore(rel, release_files) + + release_files = list(list_files(args.ftp_root, rel)) + sign_release_files_with_sigstore(args.ftp_root, rel, release_files) n = 0 file_dicts = {} - for rfile, file_desc, os_pk, add_download, add_desc in release_files: + for rfile, file_desc, os_slug, add_download, add_desc in release_files: + if not os_slug: + continue + os_pk = query_object(args.base_url, "os", slug=os_slug) file_dict = build_file_dict( - rel, rfile, rel_pk, file_desc, os_pk, add_download, add_desc + args.ftp_root, rel, rfile, rel_pk, file_desc, os_pk, add_download, add_desc ) key = file_dict["slug"] - if not os_pk: - continue print("Creating ReleaseFile object for", rfile, key) if key in file_dicts: raise RuntimeError(f"duplicate slug generated: {key}") file_dicts[key] = file_dict print("Deleting previous release files") resp = requests.delete( - base_url + f"downloads/release_file/?release={rel_pk}", headers=headers + args.base_url + f"downloads/release_file/?release={rel_pk}", headers=headers ) if resp.status_code != 204: raise RuntimeError(f"deleting previous releases failed: {resp.status_code}") for file_dict in file_dicts.values(): - file_pk = post_object("release_file", file_dict) + file_pk = post_object(args.base_url, "release_file", file_dict) if file_pk >= 0: print("Created as id =", file_pk) n += 1 diff --git a/tests/fake-ftp-files.txt b/tests/fake-ftp-files.txt index 3b2dc9f8..d3db468e 100644 --- a/tests/fake-ftp-files.txt +++ b/tests/fake-ftp-files.txt @@ -521,6 +521,7 @@ python-3.14.0b2.exe.spdx.json python-3.14.0b2t-amd64.zip python-3.14.0b2t-arm64.zip python-3.14.0b2t-win32.zip +python-3.14.0b3-aarch64-linux-android.tar.gz python-3.14.0b3-amd64.exe python-3.14.0b3-amd64.exe.crt python-3.14.0b3-amd64.exe.sig @@ -559,6 +560,7 @@ python-3.14.0b3-test-amd64.zip python-3.14.0b3-test-arm64.zip python-3.14.0b3-test-win32.zip python-3.14.0b3-win32.zip +python-3.14.0b3-x86_64-linux-android.tar.gz python-3.14.0b3.exe python-3.14.0b3.exe.crt python-3.14.0b3.exe.sig diff --git a/tests/test_add_to_pydotorg.py b/tests/test_add_to_pydotorg.py index 34a65015..e7bbaacd 100644 --- a/tests/test_add_to_pydotorg.py +++ b/tests/test_add_to_pydotorg.py @@ -83,72 +83,87 @@ def test_minor_version_tuple(release: str, expected: tuple[int, int]) -> None: def test_list_files(fs: FakeFilesystem) -> None: # Arrange + fake_ftp_root = "/fake_ftp_root" fs.add_real_file("tests/fake-ftp-files.txt") fake_files = Path("tests/fake-ftp-files.txt").read_text().splitlines() for fn in fake_files: if fn.startswith("#"): # comment continue - file_path = Path(add_to_pydotorg.ftp_root) / "3.14.0" / fn + file_path = Path(fake_ftp_root) / "3.14.0" / fn if fn.endswith("/"): fs.create_dir(file_path) else: fs.create_file(file_path) # Act - files = list(add_to_pydotorg.list_files("3.14.0b3")) + files = list(add_to_pydotorg.list_files(fake_ftp_root, "3.14.0b3")) # Assert assert files == [ - ("Python-3.14.0b3.tar.xz", "XZ compressed source tarball", 3, True, ""), - ("Python-3.14.0b3.tgz", "Gzipped source tarball", 3, False, ""), + ("Python-3.14.0b3.tar.xz", "XZ compressed source tarball", "source", True, ""), + ("Python-3.14.0b3.tgz", "Gzipped source tarball", "source", False, ""), ( + "python-3.14.0b3-aarch64-linux-android.tar.gz", + "Android embeddable package (aarch64)", + "android", + False, + "", + ), +( "python-3.14.0b3-amd64.exe", "Windows installer (64-bit)", - 1, + "windows", True, "Recommended", ), ( "python-3.14.0b3-arm64.exe", "Windows installer (ARM64)", - 1, + "windows", False, "Experimental", ), ( "python-3.14.0b3-embed-amd64.zip", "Windows embeddable package (64-bit)", - 1, + "windows", False, "", ), ( "python-3.14.0b3-embed-arm64.zip", "Windows embeddable package (ARM64)", - 1, + "windows", False, "", ), ( "python-3.14.0b3-embed-win32.zip", "Windows embeddable package (32-bit)", - 1, + "windows", False, "", ), ( "python-3.14.0b3-macos11.pkg", "macOS 64-bit universal2 installer", - 2, + "macos", True, "for macOS 10.13 and later", ), - ("python-3.14.0b3.exe", "Windows installer (32-bit)", 1, False, ""), + ( + "python-3.14.0b3-x86_64-linux-android.tar.gz", + "Android embeddable package (x86_64)", + "android", + False, + "", + ), + ("python-3.14.0b3.exe", "Windows installer (32-bit)", "windows", False, ""), ( "windows-3.14.0b3.json", "Windows release manifest", - 1, + "windows", False, "Install with 'py install 3.14'", ), From 2bc23d580e9368e9158ed102d96c6171fa8fa0aa Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 7 Aug 2025 22:45:10 +0100 Subject: [PATCH 06/20] Fix formatting --- add_to_pydotorg.py | 3 +-- tests/test_add_to_pydotorg.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/add_to_pydotorg.py b/add_to_pydotorg.py index fc5af85c..a0bf17b5 100755 --- a/add_to_pydotorg.py +++ b/add_to_pydotorg.py @@ -339,8 +339,7 @@ def sign_release_files_with_sigstore( ftp_root: str, release: str, release_files: list[tuple[str, str, int, bool, str]] ) -> None: filenames = [ - ftp_root + f"{base_version(release)}/{rfile}" - for rfile, *_ in release_files + ftp_root + f"{base_version(release)}/{rfile}" for rfile, *_ in release_files ] def has_sigstore_signature(filename: str) -> bool: diff --git a/tests/test_add_to_pydotorg.py b/tests/test_add_to_pydotorg.py index e7bbaacd..ab6dd953 100644 --- a/tests/test_add_to_pydotorg.py +++ b/tests/test_add_to_pydotorg.py @@ -110,7 +110,7 @@ def test_list_files(fs: FakeFilesystem) -> None: False, "", ), -( + ( "python-3.14.0b3-amd64.exe", "Windows installer (64-bit)", "windows", From 6efd5acd3ee8c47002e2c3006cb59c9e64d8681e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 9 Aug 2025 20:07:45 +0100 Subject: [PATCH 07/20] Apply suggestions from code review Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- add_to_pydotorg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/add_to_pydotorg.py b/add_to_pydotorg.py index a0bf17b5..dafcfcd7 100755 --- a/add_to_pydotorg.py +++ b/add_to_pydotorg.py @@ -474,12 +474,12 @@ def ensure_trailing_slash(s: str): "--ftp-root", metavar="DIR", type=ensure_trailing_slash, - default="/srv/www.python.org/ftp/python", + default="/srv/www.python.org/ftp/python/", help="FTP root; defaults to %(default)s", ) parser.add_argument( "release", - help="Python version number, e.g. 3.3.5rc2", + help="Python version number, e.g. 3.14.0rc2", ) return parser.parse_args() From 197e7ef132e5cb50caa51dbb884776ec1a0ee0b1 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 9 Aug 2025 20:14:40 +0100 Subject: [PATCH 08/20] Update to use new CI script --- .github/workflows/build-release.yml | 17 ++++++++++++----- add_to_pydotorg.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index dc0541c7..11e34134 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -41,9 +41,9 @@ permissions: {} # Set from inputs for workflow_dispatch, or set defaults to test push/PR events env: - GIT_REMOTE: ${{ github.event.inputs.git_remote || 'python' }} - GIT_COMMIT: ${{ github.event.inputs.git_commit || '4f8bb3947cfbc20f970ff9d9531e1132a9e95396' }} - CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.13.2' }} + GIT_REMOTE: ${{ github.event.inputs.git_remote || 'mhsmith' }} + GIT_COMMIT: ${{ github.event.inputs.git_commit || 'c9700e47b6bf98f6ca2e60ecdbee15ba228dcc7c' }} + CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.15.992' }} jobs: verify-input: @@ -209,6 +209,8 @@ jobs: runs-on: ubuntu-24.04 runs-on: ${{ matrix.runs-on }} + env: + triplet: ${{ matrix.arch }}-linux-android steps: - name: "Checkout ${{ env.GIT_REMOTE }}/cpython" uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -217,6 +219,11 @@ jobs: repository: "${{ env.GIT_REMOTE }}/cpython" ref: "v${{ env.CPYTHON_RELEASE }}" - - uses: ./.github/actions/build-android + - name: Build and test + run: ./Android/android.py ci "$triplet" + + - uses: actions/upload-artifact@v4 with: - triplet: ${{ matrix.arch }}-linux-android + name: ${{ env.triplet }} + path: cross-build/${{ env.triplet }}/dist/* + if-no-files-found: error diff --git a/add_to_pydotorg.py b/add_to_pydotorg.py index dafcfcd7..927601c1 100755 --- a/add_to_pydotorg.py +++ b/add_to_pydotorg.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ Script to add ReleaseFile objects for Python releases on the new pydotorg. -To use (RELEASE is something like 3.3.5rc2): +To use (RELEASE is the full Python version number): * Copy this script to dl-files (it needs access to all the release files). You could also download all files, then you need to use the "--ftp-root" From 43ad815208dc39f00a09bb59bf851815529493a6 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 9 Aug 2025 20:27:33 +0100 Subject: [PATCH 09/20] Update type annotations --- add_to_pydotorg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/add_to_pydotorg.py b/add_to_pydotorg.py index 927601c1..ab382475 100755 --- a/add_to_pydotorg.py +++ b/add_to_pydotorg.py @@ -93,7 +93,7 @@ def run_cmd( def get_file_descriptions( release: str, -) -> list[tuple[re.Pattern[str], tuple[str, int, bool, str]]]: +) -> list[tuple[re.Pattern[str], tuple[str, str, bool, str]]]: v = minor_version_tuple(release) rx = re.compile # value is (file "name", OS slug, download button, file "description"). @@ -274,7 +274,7 @@ def build_file_dict( def list_files( ftp_root: str, release: str -) -> Generator[tuple[str, str, int, bool, str], None, None]: +) -> Generator[tuple[str, str, str, bool, str], None, None]: """List all of the release's download files.""" reldir = base_version(release) for rfile in os.listdir(path.join(ftp_root, reldir)): @@ -336,7 +336,7 @@ def post_object(base_url: str, objtype: str, datadict: dict[str, Any]) -> int: def sign_release_files_with_sigstore( - ftp_root: str, release: str, release_files: list[tuple[str, str, int, bool, str]] + ftp_root: str, release: str, release_files: list[tuple[str, str, str, bool, str]] ) -> None: filenames = [ ftp_root + f"{base_version(release)}/{rfile}" for rfile, *_ in release_files @@ -457,7 +457,7 @@ def has_sigstore_signature(filename: str) -> bool: def parse_args() -> argparse.Namespace: - def ensure_trailing_slash(s: str): + def ensure_trailing_slash(s: str) -> str: if not s.endswith("/"): s += "/" return s From 6f5dc98674d974ae88b542b264f6b02a9c2963a3 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 9 Aug 2025 20:50:09 +0100 Subject: [PATCH 10/20] Revert to previous inputs --- .github/workflows/build-release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 96ab4e9e..a0390aaf 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -41,9 +41,9 @@ permissions: {} # Set from inputs for workflow_dispatch, or set defaults to test push/PR events env: - GIT_REMOTE: ${{ github.event.inputs.git_remote || 'mhsmith' }} - GIT_COMMIT: ${{ github.event.inputs.git_commit || 'c9700e47b6bf98f6ca2e60ecdbee15ba228dcc7c' }} - CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.15.992' }} + GIT_REMOTE: ${{ github.event.inputs.git_remote || 'python' }} + GIT_COMMIT: ${{ github.event.inputs.git_commit || '4f8bb3947cfbc20f970ff9d9531e1132a9e95396' }} + CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.13.2' }} jobs: verify-input: From 298c8ee117c187f7b03977ab9472c02e6cb8d8e1 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 10 Aug 2025 00:32:11 +0100 Subject: [PATCH 11/20] Add more unit tests --- tests/test_add_to_pydotorg.py | 41 ++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/test_add_to_pydotorg.py b/tests/test_add_to_pydotorg.py index ab6dd953..300f7fb4 100644 --- a/tests/test_add_to_pydotorg.py +++ b/tests/test_add_to_pydotorg.py @@ -30,16 +30,51 @@ def test_sigfile_for() -> None: @pytest.mark.parametrize( - ["release", "expected"], + ["text", "expected"], [ ("3.9.0a0", "390a0"), ("3.10.0b3", "3100b3"), ("3.11.0rc2", "3110rc2"), ("3.12.15", "31215"), + ("Hello, world!", "Hello-world"), ], ) -def test_make_slug(release: str, expected: str) -> None: - assert add_to_pydotorg.make_slug(release) == expected +def test_make_slug(text: str, expected: str) -> None: + assert add_to_pydotorg.make_slug(text) == expected + + +def test_build_file_dict(tmp_path: Path) -> None: + release = "3.14.0rc2" + release_url = "https://www.python.org/ftp/python/3.14.0" + release_dir = tmp_path / "3.14.0" + release_dir.mkdir() + + rfile = "test-artifact.txt" + (release_dir / rfile).write_text("Hello world") + (release_dir / f"{rfile}.sigstore").touch() + + assert add_to_pydotorg.build_file_dict( + str(tmp_path), + release, + rfile, + 12, + "Test artifact", + 34, + True, + "Test description", + ) == { + "name": "Test artifact", + "slug": "3140-rc2-Test-artifact", + "os": "/api/v1/downloads/os/34/", + "release": "/api/v1/downloads/release/12/", + "description": "Test description", + "is_source": False, + "url": f"{release_url}/test-artifact.txt", + "md5_sum": "3e25960a79dbc69b674cd4ec67a72c62", + "filesize": 11, + "download_button": True, + "sigstore_bundle_file": f"{release_url}/test-artifact.txt.sigstore", + } @pytest.mark.parametrize( From 35a3bcb69eda157a34fba6f02f189e95037e963b Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 10 Aug 2025 16:17:07 +0100 Subject: [PATCH 12/20] Update run_release.py --- run_release.py | 59 +++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/run_release.py b/run_release.py index d15ab031..79b6c35e 100755 --- a/run_release.py +++ b/run_release.py @@ -563,21 +563,26 @@ def create_tag(db: ReleaseShelf) -> None: ) -def wait_for_source_and_docs_artifacts(db: ReleaseShelf) -> None: - # Determine if we need to wait for docs or only source artifacts. +def wait_for_build_release(db: ReleaseShelf) -> None: + # Determine if we need to wait for docs. release_tag = db["release"] should_wait_for_docs = release_tag.includes_docs # Create the directory so it's easier to place the artifacts there. release_path = Path(db["git_repo"] / str(release_tag)) - src_path = release_path / "src" - src_path.mkdir(parents=True, exist_ok=True) + downloads_path = release_path / "downloads" + downloads_path.mkdir(parents=True, exist_ok=True) # Build the list of filepaths we're expecting. wait_for_paths = [ - src_path / f"Python-{release_tag}.tgz", - src_path / f"Python-{release_tag}.tar.xz", + downloads_path / f"Python-{release_tag}.tgz", + downloads_path / f"Python-{release_tag}.tar.xz", ] + if release_tag.as_tuple() >= (3, 14): + wait_for_paths += [ + downloads_path / f"python-{release_tag}-{arch}-linux-android.tar.gz" + for arch in ["aarch64", "x86_64"] + ] if should_wait_for_docs: docs_path = release_path / "docs" docs_path.mkdir(parents=True, exist_ok=True) @@ -595,12 +600,12 @@ def wait_for_source_and_docs_artifacts(db: ReleaseShelf) -> None: ] ) - print( - f"Waiting for source{' and docs' if should_wait_for_docs else ''} artifacts to be built" - ) - print(f"Artifacts should be placed at '{release_path}':") + print("Once the build-release workflow is complete:") + print("- Download its artifacts from the workflow summary page.") + print(f"- Copy the following files into {release_path}:") for path in wait_for_paths: - print(f"- '{os.path.relpath(path, release_path)}'") + print(f" - {os.path.relpath(path, release_path)}") + print("The script will continue once all files are present.") while not all(path.exists() for path in wait_for_paths): time.sleep(1) @@ -636,7 +641,7 @@ def sign_source_artifacts(db: ReleaseShelf) -> None: subprocess.check_call('gpg -K | grep -A 1 "^sec"', shell=True) uid = input("Please enter key ID to use for signing: ") - tarballs_path = Path(db["git_repo"] / str(db["release"]) / "src") + tarballs_path = Path(db["git_repo"] / str(db["release"]) / "downloads") tgz = str(tarballs_path / f"Python-{db['release']}.tgz") xz = str(tarballs_path / f"Python-{db['release']}.tar.xz") @@ -678,7 +683,9 @@ def build_sbom_artifacts(db: ReleaseShelf) -> None: # For each source tarball build an SBOM. for ext in (".tgz", ".tar.xz"): tarball_name = f"Python-{release_version}{ext}" - tarball_path = str(db["git_repo"] / str(db["release"]) / "src" / tarball_name) + tarball_path = str( + db["git_repo"] / str(db["release"]) / "downloads" / tarball_name + ) print(f"Building an SBOM for artifact '{tarball_name}'") sbom_data = sbom.create_sbom_for_source_tarball(tarball_path) @@ -724,7 +731,7 @@ def upload_files_to_server(db: ReleaseShelf, server: str) -> None: transport = client.get_transport() assert transport is not None, f"SSH transport to {server} is None" - destination = Path(f"/home/psf-users/{db['ssh_user']}/{db['release']}") + destination = Path(f"/home/{db['ssh_user']}/{db['release']}") ftp_client = MySFTPClient.from_transport(transport) assert ftp_client is not None, f"SFTP client to {server} is None" @@ -750,7 +757,7 @@ def upload_subdir(subdir: str) -> None: if server == DOCS_SERVER: upload_subdir("docs") elif server == DOWNLOADS_SERVER: - upload_subdir("src") + upload_subdir("downloads") if (artifacts_path / "docs").exists(): upload_subdir("docs") @@ -769,7 +776,7 @@ def place_files_in_download_folder(db: ReleaseShelf) -> None: transport = client.get_transport() assert transport is not None, f"SSH transport to {DOWNLOADS_SERVER} is None" - # Sources + # Downloads source = f"/home/psf-users/{db['ssh_user']}/{db['release']}" destination = f"/srv/www.python.org/ftp/python/{db['release'].normalized()}" @@ -781,7 +788,7 @@ def execute_command(command: str) -> None: raise ReleaseException(channel.recv_stderr(1000)) execute_command(f"mkdir -p {destination}") - execute_command(f"cp {source}/src/* {destination}") + execute_command(f"cp {source}/downloads/* {destination}") execute_command(f"chgrp downloads {destination}") execute_command(f"chmod 775 {destination}") execute_command(f"find {destination} -type f -exec chmod 664 {{}} \\;") @@ -880,7 +887,7 @@ def get_origin_remote_url(git_repo: Path) -> str: return origin_remote_url -def start_build_of_source_and_docs(db: ReleaseShelf) -> None: +def start_build_release(db: ReleaseShelf) -> None: commit_sha = get_commit_sha(db["release"].gitname, db["git_repo"]) origin_remote_url = get_origin_remote_url(db["git_repo"]) origin_remote_github_owner = extract_github_owner(origin_remote_url) @@ -922,8 +929,8 @@ def start_build_of_source_and_docs(db: ReleaseShelf) -> None: ) print() - if not ask_question("Have you started the source and docs build?"): - raise ReleaseException("Source and docs build must be started") + if not ask_question("Have you started the build-release workflow?"): + raise ReleaseException("build-release workflow must be started") def send_email_to_platform_release_managers(db: ReleaseShelf) -> None: @@ -936,7 +943,7 @@ def send_email_to_platform_release_managers(db: ReleaseShelf) -> None: print(f"{github_prefix}/{db['release'].gitname}") print(f"Git commit SHA: {commit_sha}") print( - "Source/docs build: https://github.com/python/release-tools/actions/runs/[ENTER-RUN-ID-HERE]" + "build-release workflow: https://github.com/python/release-tools/actions/runs/[ENTER-RUN-ID-HERE]" ) print() @@ -1363,18 +1370,12 @@ def _api_key(api_key: str) -> str: Task(check_cpython_repo_is_clean, "Checking Git repository is clean"), Task(create_tag, "Create tag"), Task(push_to_local_fork, "Push new tags and branches to private fork"), - Task( - start_build_of_source_and_docs, - "Start the builds for source and docs artifacts", - ), + Task(start_build_release, "Start the build-release workflow"), Task( send_email_to_platform_release_managers, "Platform release managers have been notified of the commit SHA", ), - Task( - wait_for_source_and_docs_artifacts, - "Wait for source and docs artifacts to build", - ), + Task(wait_for_build_release, "Wait for build-release workflow"), Task(check_doc_unreleased_version, "Check docs for `(unreleased)`"), Task(build_sbom_artifacts, "Building SBOM artifacts"), *([] if no_gpg else [Task(sign_source_artifacts, "Sign source artifacts")]), From fc09b3d61960d880c80fd72c2a3c4766798de209 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Aug 2025 14:18:27 +0100 Subject: [PATCH 13/20] Workflow cleanups --- .github/workflows/build-release.yml | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index a0390aaf..80f424c4 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -41,17 +41,18 @@ permissions: {} # Set from inputs for workflow_dispatch, or set defaults to test push/PR events env: - GIT_REMOTE: ${{ github.event.inputs.git_remote || 'python' }} - GIT_COMMIT: ${{ github.event.inputs.git_commit || '4f8bb3947cfbc20f970ff9d9531e1132a9e95396' }} - CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.13.2' }} + GIT_REMOTE: ${{ github.event.inputs.git_remote || 'mhsmith' }} + GIT_COMMIT: ${{ github.event.inputs.git_commit || 'c9700e47b6bf98f6ca2e60ecdbee15ba228dcc7c' }} + CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.15.992' }} jobs: verify-input: runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: - # Needed because env vars are not available in the build-docs check below + # Needed because env vars are not available in "if" conditions cpython_release: ${{ env.CPYTHON_RELEASE }} + cpython_branch: ${{ steps.get-branch.outputs.branch }} steps: - name: "Workflow run information" run: | @@ -74,6 +75,11 @@ jobs: exit 1 fi + - name: "Get CPython branch" + id: get-branch + run: | + echo "branch=$(echo "$CPYTHON_RELEASE" | cut -d . -f 1-2)" >> "$GITHUB_OUTPUT" + build-source: runs-on: ubuntu-24.04 timeout-minutes: 15 @@ -191,22 +197,13 @@ jobs: ./bin/python3 -m test -uall build-android: + name: build-android (${{ matrix.arch }}) needs: - verify-input + if: fromJSON(needs.verify-input.outputs.cpython_branch) >= 3.14 - # Android binary releases began in Python 3.14. - if: | - !( - startsWith(needs.verify-input.outputs.cpython_release, '3.9.') || - startsWith(needs.verify-input.outputs.cpython_release, '3.10.') || - startsWith(needs.verify-input.outputs.cpython_release, '3.11.') || - startsWith(needs.verify-input.outputs.cpython_release, '3.12.') || - startsWith(needs.verify-input.outputs.cpython_release, '3.13.') - ) - timeout-minutes: 60 strategy: matrix: - arch: [aarch64, x86_64] include: - arch: aarch64 runs-on: macos-15 @@ -214,6 +211,7 @@ jobs: runs-on: ubuntu-24.04 runs-on: ${{ matrix.runs-on }} + timeout-minutes: 60 env: triplet: ${{ matrix.arch }}-linux-android steps: From 0b080a090fb291b3a7f0e63264ea52ccf0bab11d Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Aug 2025 16:42:01 +0100 Subject: [PATCH 14/20] Move job selection to an external script --- .github/workflows/build-release.yml | 29 +++++++++++++---------- requirements.in | 1 + requirements.txt | 3 +++ select_jobs.py | 26 +++++++++++++++++++++ tests/test_select_jobs.py | 36 +++++++++++++++++++++++++++++ 5 files changed, 83 insertions(+), 12 deletions(-) create mode 100755 select_jobs.py create mode 100644 tests/test_select_jobs.py diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 80f424c4..54ecfe40 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -42,17 +42,16 @@ permissions: {} # Set from inputs for workflow_dispatch, or set defaults to test push/PR events env: GIT_REMOTE: ${{ github.event.inputs.git_remote || 'mhsmith' }} - GIT_COMMIT: ${{ github.event.inputs.git_commit || 'c9700e47b6bf98f6ca2e60ecdbee15ba228dcc7c' }} - CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.15.992' }} + GIT_COMMIT: ${{ github.event.inputs.git_commit || '7e87375eeff06e03a66209e506ce83c3f70ec57f' }} + CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.15.0b991' }} jobs: verify-input: runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: - # Needed because env vars are not available in "if" conditions - cpython_release: ${{ env.CPYTHON_RELEASE }} - cpython_branch: ${{ steps.get-branch.outputs.branch }} + build-docs: ${{ steps.select-jobs.outputs.docs }} + build-android: ${{ steps.select-jobs.outputs.android }} steps: - name: "Workflow run information" run: | @@ -75,10 +74,18 @@ jobs: exit 1 fi - - name: "Get CPython branch" - id: get-branch + - name: "Setup Python" + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: 3.12 + + - name: "Install dependencies" + run: python -m pip install --no-deps -r requirements.txt + + - name: "Select jobs" + id: select-jobs run: | - echo "branch=$(echo "$CPYTHON_RELEASE" | cut -d . -f 1-2)" >> "$GITHUB_OUTPUT" + ./select-jobs.py "$CPYTHON_RELEASE" >> "$GITHUB_OUTPUT" build-source: runs-on: ubuntu-24.04 @@ -126,9 +133,7 @@ jobs: timeout-minutes: 45 needs: - verify-input - - # Docs aren't built for alpha or beta releases. - if: (!(contains(needs.verify-input.outputs.cpython_release, 'a') || contains(needs.verify-input.outputs.cpython_release, 'b'))) + if: fromJSON(needs.verify-input.outputs.build-docs) steps: - name: "Checkout ${{ env.GIT_REMOTE }}/cpython" uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -200,7 +205,7 @@ jobs: name: build-android (${{ matrix.arch }}) needs: - verify-input - if: fromJSON(needs.verify-input.outputs.cpython_branch) >= 3.14 + if: fromJSON(needs.verify-input.outputs.build-android) strategy: matrix: diff --git a/requirements.in b/requirements.in index e529d6f7..a6f20bbd 100644 --- a/requirements.in +++ b/requirements.in @@ -5,4 +5,5 @@ alive_progress>=3.3.0 python-gnupg aiohttp blurb>=1.2.1 +packaging sigstore>=3 diff --git a/requirements.txt b/requirements.txt index cb64f944..580e266f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -562,6 +562,9 @@ multidict==6.1.0 \ # aiohttp # grpclib # yarl +packaging==23.2 \ + --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 + # via -r requirements.in paramiko==3.5.1 \ --hash=sha256:43b9a0501fc2b5e70680388d9346cf252cfb7d00b0667c39e80eb43a408b8f61 # via -r requirements.in diff --git a/select_jobs.py b/select_jobs.py new file mode 100755 index 00000000..b22511eb --- /dev/null +++ b/select_jobs.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +import argparse + +from packaging.version import Version + + +def output(key: str, value: bool) -> None: + print(f"{key}={str(value).lower()}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("version", type=Version) + args = parser.parse_args() + version = args.version + + # Docs are only built for stable releases or release candidates. + output("docs", version.pre is None or version.pre[0] == "rc") + + # Android binary releases began in Python 3.14. + output("android", version.release >= (3, 14)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_select_jobs.py b/tests/test_select_jobs.py new file mode 100644 index 00000000..cbb3aa37 --- /dev/null +++ b/tests/test_select_jobs.py @@ -0,0 +1,36 @@ +import sys +from textwrap import dedent + +import pytest + +import select_jobs + + +@pytest.mark.parametrize( + ("version", "docs", "android"), + [ + ("3.13.0a1", "false", "false"), + ("3.13.0rc1", "true", "false"), + ("3.13.0", "true", "false"), + ("3.13.1", "true", "false"), + ("3.14.0a1", "false", "true"), + ("3.14.0rc1", "true", "true"), + ("3.14.0", "true", "true"), + ("3.14.1", "true", "true"), + ], +) +def test_select_jobs( + version: str, + docs: str, + android: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr(sys, "argv", ["select_jobs.py", version]) + select_jobs.main() + assert capsys.readouterr().out == dedent( + f"""\ + docs={docs} + android={android} + """ + ) From 5e1f94707dc0bc652da002fd0bf97411f41361da Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Aug 2025 16:51:52 +0100 Subject: [PATCH 15/20] Add missing checkout --- .github/workflows/build-release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 54ecfe40..3e7b098e 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -59,6 +59,11 @@ jobs: echo "git_commit: $GIT_COMMIT" echo "cpython_release: $CPYTHON_RELEASE" + - name: "Checkout python/release-tools" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: "Checkout ${{ env.GIT_REMOTE }}/cpython" uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: From ad070896e8b95c438548e43cedb6bfe03887bac9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Aug 2025 16:57:27 +0100 Subject: [PATCH 16/20] Fix typo --- .github/workflows/build-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 3e7b098e..4a529b59 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -90,7 +90,7 @@ jobs: - name: "Select jobs" id: select-jobs run: | - ./select-jobs.py "$CPYTHON_RELEASE" >> "$GITHUB_OUTPUT" + ./select_jobs.py "$CPYTHON_RELEASE" >> "$GITHUB_OUTPUT" build-source: runs-on: ubuntu-24.04 From 1b15b5734d255dd4950229cd6d0251043dc1e15e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Aug 2025 17:30:56 +0100 Subject: [PATCH 17/20] Revert to previous inputs --- .github/workflows/build-release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 4a529b59..f7afead3 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -41,9 +41,9 @@ permissions: {} # Set from inputs for workflow_dispatch, or set defaults to test push/PR events env: - GIT_REMOTE: ${{ github.event.inputs.git_remote || 'mhsmith' }} - GIT_COMMIT: ${{ github.event.inputs.git_commit || '7e87375eeff06e03a66209e506ce83c3f70ec57f' }} - CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.15.0b991' }} + GIT_REMOTE: ${{ github.event.inputs.git_remote || 'python' }} + GIT_COMMIT: ${{ github.event.inputs.git_commit || '4f8bb3947cfbc20f970ff9d9531e1132a9e95396' }} + CPYTHON_RELEASE: ${{ github.event.inputs.cpython_release || '3.13.2' }} jobs: verify-input: From b20f57be2b0621e08a1129e0d3a1dcc8cafc6b8a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 13 Aug 2025 12:30:18 +0100 Subject: [PATCH 18/20] Apply suggestions from code review Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build-release.yml | 2 +- tests/test_select_jobs.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index f7afead3..a965af21 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -35,7 +35,7 @@ on: type: string description: "CPython release number (ie '3.11.5', note without the 'v' prefix)" -name: "Build Python release artifacts" +name: "Build release artifacts" permissions: {} diff --git a/tests/test_select_jobs.py b/tests/test_select_jobs.py index cbb3aa37..6d649172 100644 --- a/tests/test_select_jobs.py +++ b/tests/test_select_jobs.py @@ -13,10 +13,12 @@ ("3.13.0rc1", "true", "false"), ("3.13.0", "true", "false"), ("3.13.1", "true", "false"), - ("3.14.0a1", "false", "true"), + ("3.14.0b2", "false", "true"), ("3.14.0rc1", "true", "true"), ("3.14.0", "true", "true"), ("3.14.1", "true", "true"), + ("3.15.0a1", "false", "true"), + ("3.15.0", "true", "true"), ], ) def test_select_jobs( From bbb2899aeebf16de179d6cc627e6dd44166eead5 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 13 Aug 2025 12:33:58 +0100 Subject: [PATCH 19/20] Run select-jobs.py through `tee` --- .github/workflows/build-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index a965af21..24213e0c 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -90,7 +90,7 @@ jobs: - name: "Select jobs" id: select-jobs run: | - ./select_jobs.py "$CPYTHON_RELEASE" >> "$GITHUB_OUTPUT" + ./select_jobs.py "$CPYTHON_RELEASE" | tee -a "$GITHUB_OUTPUT" build-source: runs-on: ubuntu-24.04 From 1c45f251aa84ad3cc6301c74fc45efe88c69c5d3 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 13 Aug 2025 12:47:02 +0100 Subject: [PATCH 20/20] Use our own `Tag` class rather than `packaging` --- .github/workflows/build-release.yml | 3 --- requirements.in | 1 - requirements.txt | 3 --- select_jobs.py | 8 ++++---- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 24213e0c..f9e4fec2 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -84,9 +84,6 @@ jobs: with: python-version: 3.12 - - name: "Install dependencies" - run: python -m pip install --no-deps -r requirements.txt - - name: "Select jobs" id: select-jobs run: | diff --git a/requirements.in b/requirements.in index a6f20bbd..e529d6f7 100644 --- a/requirements.in +++ b/requirements.in @@ -5,5 +5,4 @@ alive_progress>=3.3.0 python-gnupg aiohttp blurb>=1.2.1 -packaging sigstore>=3 diff --git a/requirements.txt b/requirements.txt index 580e266f..cb64f944 100644 --- a/requirements.txt +++ b/requirements.txt @@ -562,9 +562,6 @@ multidict==6.1.0 \ # aiohttp # grpclib # yarl -packaging==23.2 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 - # via -r requirements.in paramiko==3.5.1 \ --hash=sha256:43b9a0501fc2b5e70680388d9346cf252cfb7d00b0667c39e80eb43a408b8f61 # via -r requirements.in diff --git a/select_jobs.py b/select_jobs.py index b22511eb..513b97b4 100755 --- a/select_jobs.py +++ b/select_jobs.py @@ -2,7 +2,7 @@ import argparse -from packaging.version import Version +from release import Tag def output(key: str, value: bool) -> None: @@ -11,15 +11,15 @@ def output(key: str, value: bool) -> None: def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("version", type=Version) + parser.add_argument("version", type=Tag) args = parser.parse_args() version = args.version # Docs are only built for stable releases or release candidates. - output("docs", version.pre is None or version.pre[0] == "rc") + output("docs", version.level in ["rc", "f"]) # Android binary releases began in Python 3.14. - output("android", version.release >= (3, 14)) + output("android", version.as_tuple() >= (3, 14)) if __name__ == "__main__":