metadata files routines

Author: vivus-ignis

src/gardenlinux/constants.py

@@ -155,3 +155,9 @@
 
 GLVD_BASE_URL = "https://glvd.ingress.glvd.gardnlinux.shoot.canary.k8s-hana.ondemand.com/v1"
 GL_DEB_REPO_BASE_URL = "https://packages.gardenlinux.io/gardenlinux"
+
+GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME = "gardenlinux-github-releases"
+
+# https://github.com/gardenlinux/gardenlinux/issues/3044
+# Empty string is the 'legacy' variant with traditional root fs and still needed/supported
+IMAGE_VARIANTS = ["", "_usi", "_tpm2_trustedboot"]

src/gardenlinux/github/release_notes/helpers.py

@@ -1,11 +1,25 @@
 import gzip
 import io
+import shutil
+from pathlib import Path
 
 import requests
+from git import Repo
 
 from gardenlinux.apt import DebsrcFile, GardenLinuxRepo
 from gardenlinux.apt.package_repo_info import compare_repo
-from gardenlinux.constants import GL_DEB_REPO_BASE_URL, REQUESTS_TIMEOUTS
+from gardenlinux.constants import (
+    GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME,
+    GL_DEB_REPO_BASE_URL,
+    IMAGE_VARIANTS,
+    REQUESTS_TIMEOUTS,
+)
+from gardenlinux.features import CName
+from gardenlinux.flavors import Parser as FlavorsParser
+from gardenlinux.logger import LoggerSetup
+from gardenlinux.s3 import S3Artifacts
+
+LOGGER = LoggerSetup.get_logger("gardenlinux.github.release_notes.helpers", "INFO")
 
 
 def get_package_list(gardenlinux_version):
@@ -33,3 +47,72 @@ def compare_apt_repo_versions(previous_version, current_version):
     for pkg in pkg_diffs:
         output += f"|{pkg[0]} | {pkg[1] if pkg[1] is not None else '-'} | {pkg[2] if pkg[2] is not None else '-'} |\n"
     return output
+
+
+def download_all_metadata_files(version, commitish):
+    repo = Repo(".")
+    commit = repo.commit(commitish)
+    flavors_data = commit.tree["flavors.yaml"].data_stream.read().decode("utf-8")
+    flavors = FlavorsParser(flavors_data).filter(only_publish=True)
+
+    local_dest_path = Path("s3_downloads")
+    if local_dest_path.exists():
+        shutil.rmtree(local_dest_path)
+    local_dest_path.mkdir(mode=0o755, exist_ok=False)
+
+    s3_artifacts = S3Artifacts(GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME)
+
+    commitish_short = commitish[:8]
+
+    for flavor in flavors:
+        cname = CName(flavor[1], flavor[0], "{0}-{1}".format(version, commitish_short))
+        LOGGER.debug(f"{flavor=} {version=} {commitish=}")
+        # Filter by image variants - only download if the flavor matches one of the variants
+        flavor_matches_variant = False
+        for variant_suffix in IMAGE_VARIANTS:
+            if variant_suffix == "":
+                last_part = cname.cname.split("-")[-1]
+                if "_" not in last_part:
+                    flavor_matches_variant = True
+                    break
+            elif variant_suffix in cname.cname:
+                # Specific variant (any non-empty string in IMAGE_VARIANTS)
+                flavor_matches_variant = True
+                break
+
+        if not flavor_matches_variant:
+            LOGGER.info(
+                f"Skipping flavor {cname.cname} - not matching image variants filter"
+            )
+            continue
+
+        try:
+            download_metadata_file(
+                s3_artifacts, cname.cname, version, commitish_short, local_dest_path
+            )
+        except IndexError:
+            LOGGER.warn(f"No artifacts found for flavor {cname.cname}, skipping...")
+            continue
+
+    return [str(artifact) for artifact in local_dest_path.iterdir()]
+
+
+def download_metadata_file(
+    s3_artifacts, cname, version, commitish_short, artifacts_dir
+):
+    """
+    Download metadata file (s3_metadata.yaml)
+    """
+    LOGGER.debug(
+        f"{s3_artifacts=} | {cname=} | {version=} | {commitish_short=} | {artifacts_dir=}"
+    )
+    release_object = list(
+        s3_artifacts._bucket.objects.filter(
+            Prefix=f"meta/singles/{cname}-{version}-{commitish_short}"
+        )
+    )[0]
+    LOGGER.debug(f"{release_object.bucket_name=} | {release_object.key=}")
+
+    s3_artifacts._bucket.download_file(
+        release_object.key, artifacts_dir.joinpath(f"{cname}.s3_metadata.yaml")
+    )

test-data/release_notes/s3_bucket_artifacts/ali-gardener_prod-amd64.s3_metadata.yaml

@@ -0,0 +1,188 @@
+architecture: amd64
+base_image: null
+build_committish: 75df9f401a842914563f312899ec3ce34b24515c
+build_timestamp: '2025-09-08T11:13:14.341330'
+gardenlinux_epoch: 1877
+logs: null
+modifiers:
+- log
+- sap
+- ssh
+- _legacy
+- _nopkg
+- _prod
+- _slim
+- base
+- server
+- cloud
+- ali
+- multipath
+- iscsi
+- nvme
+- gardener
+paths:
+- md5sum: 509c00f64279214b2b2da852eff090a5
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.manifest
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.manifest
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .manifest
+- md5sum: ad6ef2a783c3ff41121c4b737676e856
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.chroot.test.xml
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.chroot.test.xml
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .chroot.test.xml
+- md5sum: 82fa064efdb35805324dbd2ddb44798b
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.requirements
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.requirements
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .requirements
+- md5sum: ed2508739ef125509598fd59501cfab1
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.chroot.test.log
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.chroot.test.log
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .chroot.test.log
+- md5sum: 1f4b5a08ab62b46d129ca5d50687b0d8
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.qcow2
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.qcow2
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .qcow2
+- md5sum: 6fc647efdb84e2510c30eff2741a573c
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.manifest.log
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.manifest.log
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .manifest.log
+- md5sum: 99ffb5ea74e0799faca2737dc3bf8d89
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.qcow2.log
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.qcow2.log
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .qcow2.log
+- md5sum: 6fc647efdb84e2510c30eff2741a573c
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.requirements.log
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.requirements.log
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .requirements.log
+- md5sum: 6fc647efdb84e2510c30eff2741a573c
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.tar.log
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.tar.log
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .tar.log
+- md5sum: d2dab0252d2b9b7542a8255a1e7e4aa7
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.release
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.release
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .release
+- md5sum: 008002e0d4cf9a832756ae303b597e56
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.tar
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.tar
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .tar
+- md5sum: 6fc647efdb84e2510c30eff2741a573c
+  name: ali-gardener_prod-amd64-1877.3-75df9f40.release.log
+  s3_bucket_name: gardenlinux-github-releases
+  s3_key: objects/ali-gardener_prod-amd64-1877.3-75df9f40/ali-gardener_prod-amd64-1877.3-75df9f40.release.log
+  sha256sum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  suffix: .release.log
+platform: ali
+published_image_metadata:
+  published_alicloud_images:
+  - image_id: m-m5efm8l2bltkbloui235
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-qingdao
+  - image_id: m-2zee5ebi20ltzy5et7in
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-beijing
+  - image_id: m-8vbddy2wfex9nb29afcy
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-zhangjiakou
+  - image_id: m-hp3bx14og6cw9thujw1d
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-huhehaote
+  - image_id: m-0jlh1iq2f3bryb5okjdk
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-wulanchabu
+  - image_id: m-bp13aseh5a2wn0s5rdz6
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-hangzhou
+  - image_id: m-uf61jbe9n8a9291h4u21
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-shanghai
+  - image_id: m-gc77bfbctuzphl2bpk0o
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-nanjing
+  - image_id: m-wz9gio8m5ey0foj0g4xx
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-shenzhen
+  - image_id: m-f8zdn54v0blnsafxb1t5
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-heyuan
+  - image_id: m-7xv0q5feffsxxyttxdy9
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-guangzhou
+  - image_id: m-gw07bfbctuzphl2bpk0p
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-fuzhou
+  - image_id: m-n4a1u2avlb9pq0u5bdms
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-wuhan-lr
+  - image_id: m-2vc5saul2saa2z57h216
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-chengdu
+  - image_id: m-j6c4zk6mwb2673iq5wrz
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: cn-hongkong
+  - image_id: m-6weibwo3vrt7ar7nelc9
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: ap-northeast-1
+  - image_id: m-mj73oldn06th2vy0ymhv
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: ap-northeast-2
+  - image_id: m-t4ngrf81d0fohwq493pw
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: ap-southeast-1
+  - image_id: m-8psd64gzc1eru0qld7cc
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: ap-southeast-3
+  - image_id: m-5tsdd6k3z1vvdyyio7zn
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: ap-southeast-6
+  - image_id: m-k1aj4usnhqcssa2fpy0c
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: ap-southeast-5
+  - image_id: m-0jo6uwekvn0gnwhwnq3s
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: ap-southeast-7
+  - image_id: m-0xi8netpfc2fdwfstz3c
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: us-east-1
+  - image_id: m-rj9gwpx907qv6p6x8w45
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: us-west-1
+  - image_id: m-4hfi34x77oaeznwuulq6
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: na-south-1
+  - image_id: m-d7o2ny5xc0m3kacxjbem
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: eu-west-1
+  - image_id: m-eb39mgohcec6gaynet9l
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: me-east-1
+  - image_id: m-gw86dlqmpaugljiykx91
+    image_name: gardenlinux-ali-gardener_prod-amd64-1877.3-75df9f40
+    region_id: eu-central-1
+require_uefi: false
+s3_bucket: gardenlinux-github-releases
+s3_key: meta/singles/ali-gardener_prod-amd64-1877.3-75df9f40
+secureboot: false
+test_result: null
+version: '1877.3'