Update Python patch upgrade automation for PBS releases (#22159)

* Update Python patch upgrade automation for PBS releases

* follow redirects

* Reset PYTHON_VERSION_FULL constant

Author: Kyle-Neale

ddev/src/ddev/cli/meta/scripts/upgrade_python.py

@@ -18,10 +18,16 @@
 
 # Python.org URLs
 PYTHON_FTP_URL = "https://www.python.org/ftp/python/"
-PYTHON_MACOS_PKG_URL_TEMPLATE = "https://www.python.org/ftp/python/{version}/python-{version}-macos11.pkg"
 PYTHON_SBOM_LINUX_URL_TEMPLATE = "https://www.python.org/ftp/python/{version}/Python-{version}.tgz.spdx.json"
 PYTHON_SBOM_WINDOWS_URL_TEMPLATE = "https://www.python.org/ftp/python/{version}/python-{version}-amd64.exe.spdx.json"
 
+# Python Build Standalone (PBS) - used for macOS
+# https://github.com/astral-sh/python-build-standalone
+PBS_LATEST_RELEASE_URL = "https://api.github.com/repos/astral-sh/python-build-standalone/releases/latest"
+PBS_SHA256SUMS_URL_TEMPLATE = (
+    "https://github.com/astral-sh/python-build-standalone/releases/download/{release}/SHA256SUMS"
+)
+
 # Regex patterns for Dockerfile updates
 # Linux: ENV PYTHON3_VERSION=3.13.7 (no quotes, matches version at end of line)
 LINUX_VERSION_PATTERN = re.compile(r'(ENV PYTHON3_VERSION=)(\d+\.\d+\.\d+)$', re.MULTILINE)
@@ -248,28 +254,44 @@ def apply_substitution(pattern: re.Pattern, replace_func, error_msg: str, _docke
 
 
 def upgrade_macos_python_version(app: Application, new_version: str, tracker: ValidationTracker):
-    macos_python_file = app.repo.path / '.github' / 'workflows' / 'resolve-build-deps.yaml'
+    """
+    Update macOS Python version in the workflow file.
 
-    macos_content = read_file_safely(macos_python_file, 'macOS workflow', tracker)
-    if macos_content is None:
+    The workflow uses Python Build Standalone (PBS) from Astral. Updates PYTHON_PATCH,
+    PBS_RELEASE, and the SHA256 hashes for both macOS architectures.
+    """
+    workflow_file = app.repo.path / '.github' / 'workflows' / 'resolve-build-deps.yaml'
+    content = read_file_safely(workflow_file, 'macOS workflow', tracker)
+    if content is None:
         return
 
-    target_line = next((line for line in macos_content.splitlines() if 'PYTHON3_DOWNLOAD_URL' in line), None)
+    new_patch = new_version.split('.')[-1]
 
-    if target_line is None:
-        tracker.error(('macOS workflow',), message='Could not find PYTHON3_DOWNLOAD_URL')
+    # Get the latest PBS release and SHA256 hashes
+    pbs_info = get_pbs_release_info(app, new_version)
+    if pbs_info is None:
+        tracker.error(
+            ('macOS workflow',),
+            message=f'Could not find PBS release with Python {new_version}. '
+            'A new PBS release may not be available yet.',
+        )
         return
 
-    new_url = PYTHON_MACOS_PKG_URL_TEMPLATE.format(version=new_version)
-    indent = target_line[: target_line.index('PYTHON3_DOWNLOAD_URL')]
-    new_line = f'{indent}PYTHON3_DOWNLOAD_URL: "{new_url}"'
+    # Define replacements: (pattern, new_value)
+    replacements = [
+        (r'^(\s*PYTHON_PATCH:\s*)\d+\s*$', rf'\g<1>{new_patch}'),
+        (r'^(\s*PBS_RELEASE:\s*)\d+\s*$', rf"\g<1>{pbs_info['release']}"),
+        (r'^(\s*PBS_SHA256__aarch64:\s*)[0-9a-f]+\s*$', rf"\g<1>{pbs_info['aarch64']}"),
+        (r'^(\s*PBS_SHA256__x86_64:\s*)[0-9a-f]+\s*$', rf"\g<1>{pbs_info['x86_64']}"),
+    ]
 
-    if target_line == new_line:
-        app.display_info(f"Python version in macOS workflow is already at {new_version}")
-        return
+    for pattern, replacement in replacements:
+        content, count = re.subn(pattern, replacement, content, count=1, flags=re.MULTILINE)
+        if count == 0:
+            tracker.error(('macOS workflow',), message=f'Could not find pattern: {pattern}')
+            return
 
-    updated_content = macos_content.replace(target_line, new_line, 1)
-    write_file_safely(macos_python_file, updated_content, 'macOS workflow', tracker)
+    write_file_safely(workflow_file, content, 'macOS workflow', tracker)
 
 
 def upgrade_python_version_full_constant(app: Application, new_version: str, tracker: ValidationTracker):
@@ -336,6 +358,46 @@ def get_latest_python_version(app: Application, major_minor: str) -> str | None:
     return str(versions[-1])
 
 
+def get_pbs_release_info(app: Application, python_version: str) -> dict[str, str] | None:
+    """
+    Get the latest PBS release info with SHA256 hashes for the specified Python version.
+
+    Returns dict with 'release', 'aarch64', 'x86_64' keys, or None if not found.
+    """
+    try:
+        # Get latest PBS release tag
+        response = httpx.get(PBS_LATEST_RELEASE_URL, timeout=30)
+        response.raise_for_status()
+        release = orjson.loads(response.text).get('tag_name')
+        if not release:
+            return None
+
+        # Fetch SHA256SUMS file and extract hashes for our target files
+        sha_response = httpx.get(PBS_SHA256SUMS_URL_TEMPLATE.format(release=release), timeout=30, follow_redirects=True)
+        if sha_response.status_code != 200:
+            return None
+
+        hashes = {'release': release}
+        for arch in ('aarch64', 'x86_64'):
+            filename = f'cpython-{python_version}+{release}-{arch}-apple-darwin-install_only_stripped.tar.gz'
+            for line in sha_response.text.splitlines():
+                if filename in line:
+                    sha_hash = line.split()[0]
+                    if validate_sha256(sha_hash):
+                        hashes[arch] = sha_hash
+                    break
+
+        # Verify we found both architectures
+        if 'aarch64' not in hashes or 'x86_64' not in hashes:
+            return None
+
+        return hashes
+
+    except httpx.RequestException as e:
+        app.display_warning(f"Error fetching PBS release info: {e}")
+        return None
+
+
 def get_python_sha256_hashes(app: Application, version: str) -> dict[str, str]:
     """
     Fetch SHA256 hashes for Python release artifacts using SBOM files.

ddev/tests/cli/meta/scripts/conftest.py

@@ -192,6 +192,7 @@ def fake_repo(tmp_path_factory, config_file, local_repo, ddev, mocker):
     )
 
     # Create fake macOS workflow file for Python upgrade tests
+    # Uses Python Build Standalone (PBS) format
     write_file(
         repo_path / '.github' / 'workflows',
         'resolve-build-deps.yaml',
@@ -201,12 +202,14 @@ def fake_repo(tmp_path_factory, config_file, local_repo, ddev, mocker):
   build-macos:
     runs-on: macos-latest
     steps:
-      - name: Install Python
+      - name: Set up Python
         env:
-          PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.13.7/python-3.13.7-macos11.pkg"
-        run: |-
-          curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
-          sudo installer -pkg python3.pkg -target /
+          PYTHON_PATCH: 7
+          PBS_RELEASE: 20251202
+          PBS_SHA256__aarch64: 799a3b76240496e4472dd60ed0cd5197e04637bea7fa16af68caeb989fadcb3a
+          PBS_SHA256__x86_64: 705b39dd74490c3e9b4beb1c4f40bf802b50ba40fe085bdca635506a944d5e74
+        run: |
+          curl -fsSL -o pbs.tgz "https://github.com/astral-sh/python-build-standalone/releases/download/$PBS_RELEASE/cpython-$PYTHON_VERSION.$PYTHON_PATCH+$PBS_RELEASE-aarch64-apple-darwin-install_only_stripped.tar.gz"
 """,
     )
 

ddev/tests/cli/meta/scripts/test_upgrade_python.py

@@ -18,6 +18,14 @@ def test_update_python_version_success(fake_repo, ddev, mocker):
             'windows_amd64_sha256': '200ddff856bbff949d2cc1be42e8807c07538abd6b6966d5113a094cf628c5c5',
         },
     )
+    mocker.patch(
+        'ddev.cli.meta.scripts.upgrade_python.get_pbs_release_info',
+        return_value={
+            'release': '20251210',
+            'aarch64': 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2',
+            'x86_64': 'f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5',
+        },
+    )
 
     result = ddev('meta', 'scripts', 'upgrade-python-version')
 
@@ -46,11 +54,14 @@ def test_update_python_version_success(fake_repo, ddev, mocker):
     assert '-Hash \'200ddff856bbff949d2cc1be42e8807c07538abd6b6966d5113a094cf628c5c5\'' in contents
     assert 'ENV PYTHON_VERSION="3.13.7"' not in contents
 
-    # Verify macOS workflow was updated
+    # Verify macOS workflow was updated with PBS format
     workflow_file = fake_repo.path / '.github' / 'workflows' / 'resolve-build-deps.yaml'
     contents = workflow_file.read_text()
-    assert 'python-3.13.9-macos11.pkg' in contents
-    assert 'python-3.13.7-macos11.pkg' not in contents
+    assert 'PYTHON_PATCH: 9' in contents
+    assert 'PYTHON_PATCH: 7' not in contents
+    assert 'PBS_RELEASE: 20251210' in contents
+    assert 'PBS_SHA256__aarch64: a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' in contents
+    assert 'PBS_SHA256__x86_64: f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3b2a1f6e5' in contents
 
 
 def test_update_python_version_already_latest(fake_repo, ddev, mocker):