WIP: OCI tagging

Author: yeoldegrove

Makefile

@@ -46,6 +46,12 @@ install-test: install-dev
 test: install-test
 	$(POETRY) run pytest -k "not kms"
 
+test-debug: install-test
+	$(POETRY) run pytest -k "not kms" -vvv -s
+
+test-trace: install-test
+	$(POETRY) run pytest -k "not kms" -vvv --log-cli-level=DEBUG
+
 format: install-dev
 	$(POETRY) run black --extend-exclude test-data/gardenlinux .
 

poetry.lock

@@ -19,6 +19,9 @@ oras = { git  = "https://github.com/oras-project/oras-py.git", rev="caf8db5b2793
 python-dotenv = "^1.0.1"
 cryptography = "^44.0.0"
 boto3 = "*"
+click = "^8.2.0"
+pygments = "^2.19.1"
+opencontainers = "^0.0.14"
 
 [tool.poetry.group.dev.dependencies]
 bandit = "^1.8.3"

pyproject.toml

@@ -164,3 +164,5 @@
 
 OCI_ANNOTATION_SIGNATURE_KEY = "io.gardenlinux.oci.signature"
 OCI_ANNOTATION_SIGNED_STRING_KEY = "io.gardenlinux.oci.signed-string"
+
+GL_USER_AGENT_REGISTRY = "gardenlinux.oci.registry/1.0"

src/gardenlinux/constants.py

@@ -226,14 +226,27 @@ def get_flavor_from_cname(cname: str, get_arch: bool = True) -> str:
     # transform to flavor:
     # azure-gardener_prod_tpm2_trustedboot-amd64
 
-    platform = cname.split("-")[0]
-    features = cname.split("-")[1:-1]
-    arch = cname.split("-")[-1]
+    parts = cname.split("-")
 
+    # Extract platform, features, and architecture
+    platform = parts[0]
+
+    # If there's more than two parts (beyond platform and arch), those are features
+    if len(parts) > 2:
+        features = "-".join(parts[1:-1])  # Join all middle parts with hyphens
+    else:
+        features = ""
+
+    arch = parts[-1]
+
+    # Create the flavor string
     if get_arch:
-        return f"{platform}-{features}-{arch}"
+        flavor = f"{platform}-{features}-{arch}" if features else f"{platform}-{arch}"
     else:
-        return f"{platform}-{features}"
+        flavor = f"{platform}-{features}" if features else platform
+
+    print(f"Extracted flavor: {flavor}")
+    return flavor
 
 
 if __name__ == "__main__":

src/gardenlinux/features/__main__.py

@@ -26,13 +26,6 @@ def cli():
     type=click.Path(),
     help="Version of image",
 )
-@click.option(
-    "--commit",
-    required=False,
-    type=click.Path(),
-    default=None,
-    help="Commit of image",
-)
 @click.option(
     "--arch",
     required=True,
@@ -58,16 +51,22 @@ def cli():
     default=False,
     help="Use HTTP to communicate with the registry",
 )
+@click.option(
+    "--additional_tag",
+    required=False,
+    multiple=True,
+    help="Additional tag to push the manifest with",
+)
 def push_manifest(
     container,
     version,
-    commit,
     arch,
     cname,
     directory,
     cosign_file,
     manifest_file,
     insecure,
+    additional_tag,
 ):
     """push artifacts from a dir to a registry, get the index-entry for the manifest in return"""
     container_name = f"{container}:{version}"
@@ -77,7 +76,7 @@ def push_manifest(
         insecure=insecure,
     )
     digest = registry.push_from_dir(
-        arch, version, cname, directory, manifest_file, commit=commit
+        arch, version, cname, directory, manifest_file, additional_tag
     )
     if cosign_file:
         print(digest, file=open(cosign_file, "w"))

src/gardenlinux/oci/__main__.py

@@ -29,7 +29,11 @@
 from oras.schemas import manifest as oras_manifest_schema
 
 from gardenlinux.features import Parser
-from ..constants import OCI_ANNOTATION_SIGNATURE_KEY, OCI_ANNOTATION_SIGNED_STRING_KEY
+from ..constants import (
+    OCI_ANNOTATION_SIGNATURE_KEY,
+    OCI_ANNOTATION_SIGNED_STRING_KEY,
+    GL_USER_AGENT_REGISTRY,
+)
 from .checksum import (
     calculate_sha256,
     verify_sha256,
@@ -653,40 +657,59 @@ def push_from_dir(
         cname: str,
         directory: str,
         manifest_file: str,
-        commit: Optional[str] = None,
+        additional_tags: list = None,
     ):
-        # Step 1 scan and extract nested artifacts:
-        for file in os.listdir(directory):
-            try:
-                if file.endswith(".pxe.tar.gz"):
-                    logger.info(f"Found nested artifact {file}")
-                    nested_tar_obj = tarfile.open(f"{directory}/{file}")
-                    nested_tar_obj.extractall(filter="data", path=directory)
-                    nested_tar_obj.close()
-            except (OSError, tarfile.FilterError, tarfile.TarError) as e:
-                print(f"Failed to extract nested artifact {file}", e)
-                exit(1)
+        """
+        Push artifacts from a directory to a registry
+
+        Args:
+            architecture: Target architecture of the image
+            version: Version tag for the image
+            cname: Canonical name of the image
+            directory: Directory containing the artifacts
+            manifest_file: File to write the manifest index entry to
+            additional_tags: Additional tags to push the manifest with
+
+        Returns:
+            The digest of the pushed manifest
+        """
+        if additional_tags is None:
+            additional_tags = []
 
         try:
+            # Step 1: scan and extract nested artifacts
+            for file in os.listdir(directory):
+                try:
+                    if file.endswith(".pxe.tar.gz"):
+                        logger.info(f"Found nested artifact {file}")
+                        nested_tar_obj = tarfile.open(f"{directory}/{file}")
+                        nested_tar_obj.extractall(filter="data", path=directory)
+                        nested_tar_obj.close()
+                except (OSError, tarfile.FilterError, tarfile.TarError) as e:
+                    print(f"Failed to extract nested artifact {file}", e)
+                    exit(1)
+
+            # Step 2: Get metadata from files
             oci_metadata = get_oci_metadata_from_fileset(
                 os.listdir(directory), architecture
             )
 
             features = ""
+            commit = ""
             for artifact in oci_metadata:
                 if artifact["media_type"] == "application/io.gardenlinux.release":
-                    file = open(f"{directory}/{artifact["file_name"]}", "r")
-                    lines = file.readlines()
-                    for line in lines:
-                        if line.strip().startswith("GARDENLINUX_FEATURES="):
-                            features = line.strip().removeprefix(
-                                "GARDENLINUX_FEATURES="
-                            )
-                            break
-                    file.close()
-
-            flavor = Parser.get_flavor_from_cname(cname, get_arch=True)
-
+                    with open(f"{directory}/{artifact["file_name"]}", "r") as file:
+                        for line in file:
+                            line = line.strip()
+                            if line.startswith("GARDENLINUX_FEATURES="):
+                                features = line.removeprefix("GARDENLINUX_FEATURES=")
+                            elif line.startswith("GARDENLINUX_COMMIT_ID="):
+                                commit = line.removeprefix("GARDENLINUX_COMMIT_ID=")
+                            if features and commit:  # Break if both values are found
+                                break
+                    break  # Break after processing the release file
+
+            # Step 3: Push the image manifest
             digest = self.push_image_manifest(
                 architecture,
                 cname,
@@ -697,7 +720,116 @@ def push_from_dir(
                 manifest_file,
                 commit=commit,
             )
+
+            # Step 4: Process additional tags if provided
+            if additional_tags and len(additional_tags) > 0:
+                print(f"DEBUG: Processing {len(additional_tags)} additional tags")
+                logger.info(f"Processing {len(additional_tags)} additional tags")
+
+                # Call push_additional_tags_manifest with repository information
+                self.push_additional_tags_manifest(
+                    architecture,
+                    cname,
+                    version,
+                    additional_tags,
+                    container=self.container,
+                )
+
+            return digest
         except Exception as e:
             print("Error: ", e)
             exit(1)
-        return digest
+
+    def push_additional_tags_manifest(
+        self, architecture, cname, version, additional_tags, container
+    ):
+        """
+        Push additional tags for an existing manifest using ORAS Registry methods
+
+        Args:
+            architecture: Target architecture of the image
+            cname: Canonical name of the image
+            version: Version tag for the image
+            additional_tags: List of additional tags to push
+            container: Container object
+        """
+        try:
+            print(f"DEBUG: Processing {len(additional_tags)} additional tags for manifest")
+            print(f"DEBUG: Container: {container}")
+            print(f"DEBUG: Container api_prefix: {container.api_prefix}")
+            print(f"DEBUG: Container uri: {container.uri}")
+            print(f"DEBUG: Container tag: {container.tag}")
+            print(f"DEBUG: Container registry: {container.registry}")
+            print(f"DEBUG: Container repository: {container.repository}")
+
+            # Source tag is the tag containing the version-cname-architecture combination
+            source_tag = f"{version}-{cname}-{architecture}"
+            source_container = copy.deepcopy(container)
+            source_container.tag = source_tag
+
+            # Authentication credentials from environment
+            token = os.getenv("GL_CLI_REGISTRY_TOKEN")
+            username = os.getenv("GL_CLI_REGISTRY_USERNAME")
+            password = os.getenv("GL_CLI_REGISTRY_PASSWORD")
+
+            print(f"DEBUG: Token is {'set' if token else 'not set'}")
+            print(f"DEBUG: Username is {'set' if username else 'not set'}")
+            print(f"DEBUG: Password is {'set' if password else 'not set'}")
+
+            # Login to registry if credentials are provided
+            if username and password:
+                print(f"DEBUG: Logging in with username/password")
+                try:
+                    self.login(username, password)
+                except Exception as login_error:
+                    print(f"DEBUG: Login error: {str(login_error)}")
+            elif token:
+                # If token is provided, set it directly on the Registry instance
+                print(f"DEBUG: Using token authentication")
+                self.token = base64.b64encode(token.encode("utf-8")).decode("utf-8")
+                self.auth.set_token_auth(self.token)
+
+            # Step 1: Get the manifest from the source container
+            try:
+                print(f"DEBUG: Getting manifest from {source_container}")
+                manifest = self.get_manifest(source_container)
+                if not manifest:
+                    print(f"DEBUG: Failed to get manifest for {source_container}")
+                    logger.error(f"Failed to get manifest for {source_container}")
+                    return
+                print(f"DEBUG: Successfully retrieved manifest: {manifest['mediaType'] if 'mediaType' in manifest else 'unknown'}")
+            except Exception as get_error:
+                print(f"DEBUG: Error getting manifest: {str(get_error)}")
+                logger.error(f"Error getting manifest: {str(get_error)}")
+                return
+
+            # Step 2: For each additional tag, push the manifest using Registry.upload_manifest
+            for tag in additional_tags:
+                try:
+                    print(f"DEBUG: Pushing additional tag: {tag}")
+
+                    # Create a new container for this tag
+                    tag_container = copy.deepcopy(container)
+                    tag_container.tag = tag
+
+                    print(f"DEBUG: Pushing to container: {tag_container}")
+
+                    # Upload the manifest to the new tag
+                    response = self.upload_manifest(manifest, tag_container)
+
+                    if response and response.status_code in [200, 201]:
+                        print(f"DEBUG: Successfully pushed tag {tag} for manifest")
+                        logger.info(f"Successfully pushed tag {tag} for manifest")
+                    else:
+                        status_code = getattr(response, 'status_code', 'unknown')
+                        response_text = getattr(response, 'text', 'No response text')
+                        print(f"DEBUG: Failed to push tag {tag} for manifest: {status_code} - {response_text}")
+                        logger.error(f"Failed to push tag {tag} for manifest: {status_code}")
+
+                except Exception as tag_error:
+                    print(f"DEBUG: Error pushing tag {tag} for manifest: {str(tag_error)}")
+                    logger.error(f"Error pushing tag {tag} for manifest: {str(tag_error)}")
+
+        except Exception as e:
+            print(f"DEBUG: Error in push_additional_tags_manifest: {str(e)}")
+            logger.error(f"Error in push_additional_tags_manifest: {str(e)}")