WIP: OCI tagging

Author: yeoldegrove

Makefile

@@ -46,6 +46,12 @@ install-test: install-dev
 test: install-test
 	$(POETRY) run pytest -k "not kms"
 
+test-debug: install-test
+	$(POETRY) run pytest -k "not kms" -vvv -s
+
+test-trace: install-test
+	$(POETRY) run pytest -k "not kms" -vvv --log-cli-level=DEBUG
+
 format: install-dev
 	$(POETRY) run black --extend-exclude test-data/gardenlinux .
 

poetry.lock

@@ -19,6 +19,9 @@ oras = { git  = "https://github.com/oras-project/oras-py.git", rev="caf8db5b2793
 python-dotenv = "^1.0.1"
 cryptography = "^44.0.0"
 boto3 = "*"
+click = "^8.2.0"
+pygments = "^2.19.1"
+opencontainers = "^0.0.14"
 
 [tool.poetry.group.dev.dependencies]
 bandit = "^1.8.3"

pyproject.toml

@@ -164,3 +164,5 @@
 
 OCI_ANNOTATION_SIGNATURE_KEY = "io.gardenlinux.oci.signature"
 OCI_ANNOTATION_SIGNED_STRING_KEY = "io.gardenlinux.oci.signed-string"
+
+GL_USER_AGENT_REGISTRY = "gardenlinux.oci.registry/1.0"

src/gardenlinux/constants.py

@@ -226,14 +226,27 @@ def get_flavor_from_cname(cname: str, get_arch: bool = True) -> str:
     # transform to flavor:
     # azure-gardener_prod_tpm2_trustedboot-amd64
 
-    platform = cname.split("-")[0]
-    features = cname.split("-")[1:-1]
-    arch = cname.split("-")[-1]
+    parts = cname.split("-")
 
+    # Extract platform, features, and architecture
+    platform = parts[0]
+
+    # If there's more than two parts (beyond platform and arch), those are features
+    if len(parts) > 2:
+        features = "-".join(parts[1:-1])  # Join all middle parts with hyphens
+    else:
+        features = ""
+
+    arch = parts[-1]
+
+    # Create the flavor string
     if get_arch:
-        return f"{platform}-{features}-{arch}"
+        flavor = f"{platform}-{features}-{arch}" if features else f"{platform}-{arch}"
     else:
-        return f"{platform}-{features}"
+        flavor = f"{platform}-{features}" if features else platform
+
+    print(f"Extracted flavor: {flavor}")
+    return flavor
 
 
 if __name__ == "__main__":

src/gardenlinux/features/__main__.py

@@ -26,13 +26,6 @@ def cli():
     type=click.Path(),
     help="Version of image",
 )
-@click.option(
-    "--commit",
-    required=False,
-    type=click.Path(),
-    default=None,
-    help="Commit of image",
-)
 @click.option(
     "--arch",
     required=True,
@@ -58,16 +51,22 @@ def cli():
     default=False,
     help="Use HTTP to communicate with the registry",
 )
+@click.option(
+    "--additional_tag",
+    required=False,
+    multiple=True,
+    help="Additional tag to push the manifest with",
+)
 def push_manifest(
     container,
     version,
-    commit,
     arch,
     cname,
     directory,
     cosign_file,
     manifest_file,
     insecure,
+    additional_tag,
 ):
     """push artifacts from a dir to a registry, get the index-entry for the manifest in return"""
     container_name = f"{container}:{version}"
@@ -77,7 +76,7 @@ def push_manifest(
         insecure=insecure,
     )
     digest = registry.push_from_dir(
-        arch, version, cname, directory, manifest_file, commit=commit
+        arch, version, cname, directory, manifest_file, additional_tag
     )
     if cosign_file:
         print(digest, file=open(cosign_file, "w"))

src/gardenlinux/oci/__main__.py

@@ -16,6 +16,7 @@
 from typing import Optional, Tuple
 
 import jsonschema
+import opencontainers.distribution.reggie as reggie
 import oras.auth
 import oras.client
 import oras.defaults
@@ -29,7 +30,11 @@
 from oras.schemas import manifest as oras_manifest_schema
 
 from gardenlinux.features import Parser
-from ..constants import OCI_ANNOTATION_SIGNATURE_KEY, OCI_ANNOTATION_SIGNED_STRING_KEY
+from ..constants import (
+    OCI_ANNOTATION_SIGNATURE_KEY,
+    OCI_ANNOTATION_SIGNED_STRING_KEY,
+    GL_USER_AGENT_REGISTRY,
+)
 from .checksum import (
     calculate_sha256,
     verify_sha256,
@@ -653,40 +658,59 @@ def push_from_dir(
         cname: str,
         directory: str,
         manifest_file: str,
-        commit: Optional[str] = None,
+        additional_tags: list = None,
     ):
-        # Step 1 scan and extract nested artifacts:
-        for file in os.listdir(directory):
-            try:
-                if file.endswith(".pxe.tar.gz"):
-                    logger.info(f"Found nested artifact {file}")
-                    nested_tar_obj = tarfile.open(f"{directory}/{file}")
-                    nested_tar_obj.extractall(filter="data", path=directory)
-                    nested_tar_obj.close()
-            except (OSError, tarfile.FilterError, tarfile.TarError) as e:
-                print(f"Failed to extract nested artifact {file}", e)
-                exit(1)
+        """
+        Push artifacts from a directory to a registry
+
+        Args:
+            architecture: Target architecture of the image
+            version: Version tag for the image
+            cname: Canonical name of the image
+            directory: Directory containing the artifacts
+            manifest_file: File to write the manifest index entry to
+            additional_tags: Additional tags to push the manifest with
+
+        Returns:
+            The digest of the pushed manifest
+        """
+        if additional_tags is None:
+            additional_tags = []
 
         try:
+            # Step 1: scan and extract nested artifacts
+            for file in os.listdir(directory):
+                try:
+                    if file.endswith(".pxe.tar.gz"):
+                        logger.info(f"Found nested artifact {file}")
+                        nested_tar_obj = tarfile.open(f"{directory}/{file}")
+                        nested_tar_obj.extractall(filter="data", path=directory)
+                        nested_tar_obj.close()
+                except (OSError, tarfile.FilterError, tarfile.TarError) as e:
+                    print(f"Failed to extract nested artifact {file}", e)
+                    exit(1)
+
+            # Step 2: Get metadata from files
             oci_metadata = get_oci_metadata_from_fileset(
                 os.listdir(directory), architecture
             )
 
             features = ""
+            commit = ""
             for artifact in oci_metadata:
                 if artifact["media_type"] == "application/io.gardenlinux.release":
-                    file = open(f"{directory}/{artifact["file_name"]}", "r")
-                    lines = file.readlines()
-                    for line in lines:
-                        if line.strip().startswith("GARDENLINUX_FEATURES="):
-                            features = line.strip().removeprefix(
-                                "GARDENLINUX_FEATURES="
-                            )
-                            break
-                    file.close()
-
-            flavor = Parser.get_flavor_from_cname(cname, get_arch=True)
-
+                    with open(f"{directory}/{artifact["file_name"]}", "r") as file:
+                        for line in file:
+                            line = line.strip()
+                            if line.startswith("GARDENLINUX_FEATURES="):
+                                features = line.removeprefix("GARDENLINUX_FEATURES=")
+                            elif line.startswith("GARDENLINUX_COMMIT_ID="):
+                                commit = line.removeprefix("GARDENLINUX_COMMIT_ID=")
+                            if features and commit:  # Break if both values are found
+                                break
+                    break  # Break after processing the release file
+
+            # Step 3: Push the image manifest
             digest = self.push_image_manifest(
                 architecture,
                 cname,
@@ -697,7 +721,165 @@ def push_from_dir(
                 manifest_file,
                 commit=commit,
             )
+
+            # Step 4: Process additional tags if provided
+            if additional_tags and len(additional_tags) > 0:
+                print(f"DEBUG: Processing {len(additional_tags)} additional tags")
+                logger.info(f"Processing {len(additional_tags)} additional tags")
+
+                # Call push_additional_tags_manifest with repository information
+                self.push_additional_tags_manifest(
+                    architecture,
+                    cname,
+                    version,
+                    additional_tags,
+                    container=self.container,
+                )
+
+            return digest
         except Exception as e:
             print("Error: ", e)
             exit(1)
-        return digest
+
+    def push_additional_tags_manifest(
+        self, architecture, cname, version, additional_tags, container
+    ):
+        """
+        Push additional tags for an existing manifest using Reggie client
+
+        Args:
+            architecture: Target architecture of the image
+            cname: Canonical name of the image
+            version: Version tag for the image
+            additional_tags: List of additional tags to push
+            container: Container object
+        """
+        try:
+            print(
+                f"DEBUG: Processing {len(additional_tags)} additional tags for manifest"
+            )
+            print(f"DEBUG: Container: {container}")
+            print(f"DEBUG: Container api_prefix: {container.api_prefix}")
+            print(f"DEBUG: Container uri: {container.uri}")
+            print(f"DEBUG: Container tag: {container.tag}")
+            print(f"DEBUG: Container registry: {container.registry}")
+            print(f"DEBUG: Container repository: {container.repository}")
+
+            # Source tag is version-cname-architecture as pushed by push_image_manifest
+            source_tag = f"{version}-{cname}-{architecture}"
+            manifest_url = container.manifest_url()
+
+            repo_name = manifest_url[manifest_url.find('/v2/')+3:manifest_url.find('/manifests/')]
+            if repo_name.startswith('/'):
+                repo_name = repo_name[1:]
+
+            # Ensure registry_url and manifest_url have protocol prefix
+            use_insecure = getattr(
+                self, "insecure", True
+            )  # Default to True if attribute doesn't exist
+            protocol = "http" if use_insecure else "https"
+            registry_url = f"{protocol}://{container.registry}"
+
+            print(f"DEBUG: Using source tag: {container.tag}")
+            print(f"DEBUG: Using registry URL: {registry_url}")
+            print(f"DEBUG: Using repository name: {repo_name}")
+            logger.info(f"Using source tag: {container.tag}")
+            logger.info(f"Using registry URL: {registry_url}")
+            logger.info(f"Using repository name: {repo_name}")
+
+            # Configure client options
+            client_options = [reggie.WithUserAgent(GL_USER_AGENT_REGISTRY)]
+
+            # Initialize reggie client
+            client = reggie.NewClient(f"{registry_url}/{repo_name}", *client_options)
+
+            # Set authentication token if provided
+            token = os.getenv("GL_CLI_REGISTRY_TOKEN")
+            username = os.getenv("GL_CLI_REGISTRY_USERNAME")
+            password = os.getenv("GL_CLI_REGISTRY_PASSWORD")
+
+            # Step 1: Get the manifest from the source tag using Reggie
+            get_manifest_req = client.NewRequest(
+                "GET", f"/v2/{repo_name}/manifests/{source_tag}"
+            )
+
+            # Set authentication for the GET request
+            if token:
+                get_manifest_req.SetAuthToken(token)
+            elif username and password:
+                get_manifest_req.SetBasicAuth(username, password)
+
+            get_manifest_resp = client.Do(get_manifest_req)
+
+            if get_manifest_resp.status_code != 200:
+                print(
+                    f"DEBUG: Failed to get source manifest: {get_manifest_resp.status_code}"
+                    f"DEBUG: Failed to get source manifest: {get_manifest_resp.text}"
+                )
+                logger.error(
+                    f"Failed to get source manifest: {get_manifest_resp.status_code}"
+                )
+                return
+
+            # Get the manifest content and content-type
+            manifest_content = get_manifest_resp.content
+            content_type = get_manifest_resp.headers.get("Content-Type")
+
+            print(
+                f"DEBUG: Successfully retrieved manifest with content: {manifest_content}"
+                f"DEBUG: Successfully retrieved manifest with content type: {content_type}"
+
+            )
+            logger.info(
+                f"Successfully retrieved manifest with content: {manifest_content}"
+                f"Successfully retrieved manifest with content type: {content_type}"
+            )
+
+            # Step 2: For each additional tag, push the manifest using Reggie
+            for tag in additional_tags:
+                try:
+                    print(f"DEBUG: Pushing additional tag: {tag}")
+
+                    # Create a new PUT request for the tag
+                    put_manifest_req = client.NewRequest(
+                        "PUT", f"/v2/{repo_name}/manifests/{tag}"
+                    )
+
+                    # Set the content type header to match the original manifest
+                    put_manifest_req.SetHeader("Content-Type", content_type)
+
+                    # Set authentication for the PUT request
+                    if token:
+                        put_manifest_req.SetAuthToken(token)
+                    elif username and password:
+                        put_manifest_req.SetBasicAuth(username, password)
+
+                    # Set the body to the manifest content
+                    put_manifest_req.SetBody(manifest_content)
+
+                    print(f"DEBUG: put_manifest_req: {put_manifest_req}")
+                    print(f"DEBUG: put_manifest_req.body: {put_manifest_req.body}")
+                    print(f"DEBUG: put_manifest_req.headers: {put_manifest_req.headers}")
+
+                    # Use the reggie client to make the request
+                    put_manifest_resp = client.Do(put_manifest_req)
+
+                    if put_manifest_resp.status_code in [200, 201]:
+                        print(f"DEBUG: Successfully pushed tag {tag} for manifest")
+                        logger.info(f"Successfully pushed tag {tag} for manifest")
+                    else:
+                        print(
+                            f"DEBUG: Failed to push tag {tag} for manifest: {put_manifest_resp.status_code}"
+                            f"DEBUG: Failed to push tag {tag} for manifest: {put_manifest_resp.text}"
+                        )
+                        logger.error(
+                            f"Failed to push tag {tag} for manifest: {put_manifest_resp.status_code}"
+                        )
+
+                except Exception as e:
+                    print(f"DEBUG: Error pushing tag {tag} for manifest: {str(e)}")
+                    logger.error(f"Error pushing tag {tag} for manifest: {str(e)}")
+
+        except Exception as e:
+            print(f"DEBUG: Error in push_additional_tags_manifest: {str(e)}")
+            logger.error(f"Error in push_additional_tags_manifest: {str(e)}")