WIP: OCI tagging

Author: yeoldegrove

Makefile

@@ -46,6 +46,12 @@ install-test: install-dev
 test: install-test
 	$(POETRY) run pytest -k "not kms"
 
+test-debug: install-test
+	$(POETRY) run pytest -k "not kms" -vvv -s
+
+test-trace: install-test
+	$(POETRY) run pytest -k "not kms" -vvv --log-cli-level=DEBUG
+
 format: install-dev
 	$(POETRY) run black --extend-exclude test-data/gardenlinux .
 

poetry.lock

@@ -19,6 +19,7 @@ oras = { git  = "https://github.com/oras-project/oras-py.git", rev="caf8db5b2793
 python-dotenv = "^1.0.1"
 cryptography = "^44.0.0"
 boto3 = "*"
+click = "^8.2.0"
 
 [tool.poetry.group.dev.dependencies]
 bandit = "^1.8.3"

pyproject.toml

@@ -226,15 +226,27 @@ def get_flavor_from_cname(cname: str, get_arch: bool = True) -> str:
     # transform to flavor:
     # azure-gardener_prod_tpm2_trustedboot-amd64
 
-    platform = cname.split("-")[0]
-    features = cname.split("-")[1:-1]
-    arch = cname.split("-")[-1]
+    parts = cname.split("-")
 
+    # Extract platform, features, and architecture
+    platform = parts[0]
+
+    # If there's more than two parts (beyond platform and arch), those are features
+    if len(parts) > 2:
+        features = "-".join(parts[1:-1])  # Join all middle parts with hyphens
+    else:
+        features = ""
+
+    arch = parts[-1]
+
+    # Create the flavor string
     if get_arch:
-        return f"{platform}-{features}-{arch}"
+        flavor = f"{platform}-{features}-{arch}" if features else f"{platform}-{arch}"
     else:
-        return f"{platform}-{features}"
+        flavor = f"{platform}-{features}" if features else platform
 
+    print(f"Extracted flavor: {flavor}")
+    return flavor
 
 if __name__ == "__main__":
     main()

src/gardenlinux/features/__main__.py

@@ -26,13 +26,6 @@ def cli():
     type=click.Path(),
     help="Version of image",
 )
-@click.option(
-    "--commit",
-    required=False,
-    type=click.Path(),
-    default=None,
-    help="Commit of image",
-)
 @click.option(
     "--arch",
     required=True,
@@ -58,16 +51,22 @@ def cli():
     default=False,
     help="Use HTTP to communicate with the registry",
 )
+@click.option(
+    "--additional_tags",
+    required=False,
+    multiple=True,
+    help="Additional tags to push the manifest with",
+)
 def push_manifest(
     container,
     version,
-    commit,
     arch,
     cname,
     directory,
     cosign_file,
     manifest_file,
     insecure,
+    additional_tags,
 ):
     """push artifacts from a dir to a registry, get the index-entry for the manifest in return"""
     container_name = f"{container}:{version}"
@@ -77,7 +76,7 @@ def push_manifest(
         insecure=insecure,
     )
     digest = registry.push_from_dir(
-        arch, version, cname, directory, manifest_file, commit=commit
+        arch, version, cname, directory, manifest_file, additional_tags
     )
     if cosign_file:
         print(digest, file=open(cosign_file, "w"))

src/gardenlinux/oci/__main__.py

@@ -651,40 +651,59 @@ def push_from_dir(
         cname: str,
         directory: str,
         manifest_file: str,
-        commit: Optional[str] = None,
+        additional_tags: list = None,
     ):
-        # Step 1 scan and extract nested artifacts:
-        for file in os.listdir(directory):
-            try:
-                if file.endswith(".pxe.tar.gz"):
-                    logger.info(f"Found nested artifact {file}")
-                    nested_tar_obj = tarfile.open(f"{directory}/{file}")
-                    nested_tar_obj.extractall(filter="data", path=directory)
-                    nested_tar_obj.close()
-            except (OSError, tarfile.FilterError, tarfile.TarError) as e:
-                print(f"Failed to extract nested artifact {file}", e)
-                exit(1)
+        """
+        Push artifacts from a directory to a registry
+
+        Args:
+            architecture: Target architecture of the image
+            version: Version tag for the image
+            cname: Canonical name of the image
+            directory: Directory containing the artifacts
+            manifest_file: File to write the manifest index entry to
+            additional_tags: Additional tags to push the manifest with
+
+        Returns:
+            The digest of the pushed manifest
+        """
+        if additional_tags is None:
+            additional_tags = []
 
         try:
+            # Step 1: scan and extract nested artifacts
+            for file in os.listdir(directory):
+                try:
+                    if file.endswith(".pxe.tar.gz"):
+                        logger.info(f"Found nested artifact {file}")
+                        nested_tar_obj = tarfile.open(f"{directory}/{file}")
+                        nested_tar_obj.extractall(filter="data", path=directory)
+                        nested_tar_obj.close()
+                except (OSError, tarfile.FilterError, tarfile.TarError) as e:
+                    print(f"Failed to extract nested artifact {file}", e)
+                    exit(1)
+
+            # Step 2: Get metadata from files
             oci_metadata = get_oci_metadata_from_fileset(
                 os.listdir(directory), architecture
             )
 
             features = ""
+            commit = ""
             for artifact in oci_metadata:
                 if artifact["media_type"] == "application/io.gardenlinux.release":
-                    file = open(f"{directory}/{artifact["file_name"]}", "r")
-                    lines = file.readlines()
-                    for line in lines:
-                        if line.strip().startswith("GARDENLINUX_FEATURES="):
-                            features = line.strip().removeprefix(
-                                "GARDENLINUX_FEATURES="
-                            )
-                            break
-                    file.close()
-
-            flavor = get_flavor_from_cname(cname, get_arch=True)
-
+                    with open(f"{directory}/{artifact["file_name"]}", "r") as file:
+                        for line in file:
+                            line = line.strip()
+                            if line.startswith("GARDENLINUX_FEATURES="):
+                                features = line.removeprefix("GARDENLINUX_FEATURES=")
+                            elif line.startswith("GARDENLINUX_COMMIT_ID="):
+                                commit = line.removeprefix("GARDENLINUX_COMMIT_ID=")
+                            if features and commit:  # Break if both values are found
+                                break
+                    break  # Break after processing the release file
+
+            # Step 3: Push the image manifest
             digest = self.push_image_manifest(
                 architecture,
                 cname,
@@ -695,11 +714,153 @@ def push_from_dir(
                 manifest_file,
                 commit=commit,
             )
+
+            # Step 4: Process additional tags if provided
+            if additional_tags and len(additional_tags) > 0:
+                print(f"DEBUG: Processing {len(additional_tags)} additional tags")
+                logger.info(f"Processing {len(additional_tags)} additional tags")
+
+                # Call push_additional_tags with repository information
+                self.push_additional_tags(
+                    architecture,
+                    cname,
+                    version,
+                    additional_tags,
+                    repo_name=None,
+                    registry_url=None
+                )
+
+            return digest
         except Exception as e:
             print("Error: ", e)
             exit(1)
-        return digest
 
+    def push_additional_tags(self, architecture, cname, version, additional_tags, repo_name=None, registry_url=None):
+        """
+        Push additional tags for an existing manifest using requests directly
+
+        Args:
+            architecture: Target architecture of the image
+            cname: Canonical name of the image
+            version: Version tag for the image
+            additional_tags: List of additional tags to push
+            repo_name: Repository name (optional, will be determined if not provided)
+            registry_url: Registry URL (optional, will be determined if not provided)
+        """
+        try:
+            print(f"DEBUG: Processing {len(additional_tags)} additional tags")
+
+            # Source tag is version-cname-architecture
+            source_tag = f"{version}-{cname}-{architecture}"
+
+            # Use provided repo_name or determine it from container_name
+            if repo_name is None:
+                # Get container parts
+                container_parts = self.container_name.split(':')
+                container_repo = container_parts[0]
+
+                # Parse repository name
+                if '/' in container_repo:
+                    # Production environment
+                    repo_parts = container_repo.split('/')
+                    repo_name = '/'.join(repo_parts[1:])
+                else:
+                    # Test environment
+                    repo_name = "gardenlinux-example"
+
+            # Use provided registry_url or get from class
+            if registry_url is None:
+                # Check if we have a registry_url property
+                if hasattr(self, 'registry_url') and self.registry_url:
+                    registry_url = self.registry_url
+                else:
+                    # Determine if we should use HTTP or HTTPS
+                    use_insecure = getattr(self, 'insecure', True)  # Default to True for tests
+                    protocol = "http" if use_insecure else "https"
+
+                    # Get registry host
+                    container_parts = self.container_name.split(':')
+                    container_repo = container_parts[0]
+
+                    if '/' in container_repo:
+                        registry_host = container_repo.split('/')[0]
+                    else:
+                        registry_host = "127.0.0.1:18081"  # Default for test environment
+
+                    registry_url = f"{protocol}://{registry_host}"
+
+            # Ensure registry_url has protocol prefix
+            if not registry_url.startswith("http://") and not registry_url.startswith("https://"):
+                use_insecure = getattr(self, 'insecure', True)  # Default to True for tests
+                protocol = "http" if use_insecure else "https"
+                registry_url = f"{protocol}://{registry_url}"
+
+            print(f"DEBUG: Using source tag: {source_tag}")
+            print(f"DEBUG: Using repository: {repo_name}")
+            print(f"DEBUG: Using registry URL: {registry_url}")
+
+            # Import requests for HTTP operations
+            import requests
+
+            # Set up authentication headers if needed
+            headers = {}
+            if hasattr(self, 'token') and self.token:
+                headers["Authorization"] = f"Bearer {self.token}"
+
+            # Add accept headers for manifest formats
+            headers.update({
+                "Accept": "application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json"
+            })
+
+            # Step 1: Get the manifest from the source tag
+            source_url = f"{registry_url}/v2/{repo_name}/manifests/{source_tag}"
+            verify = not getattr(self, 'insecure', True)  # Default to not verify for tests
+
+            # Get the source manifest
+            source_resp = requests.get(source_url, headers=headers, verify=verify)
+
+            if source_resp.status_code != 200:
+                print(f"DEBUG: Failed to get source manifest: {source_resp.status_code}")
+                logger.error(f"Failed to get source manifest: {source_resp.status_code}")
+                return
+
+            # Get the manifest content and content-type
+            manifest_content = source_resp.text
+            content_type = source_resp.headers.get("Content-Type", "application/vnd.oci.image.manifest.v1+json")
+
+            print(f"DEBUG: Successfully retrieved manifest with content type: {content_type}")
+
+            # Step 2: For each additional tag, push the manifest
+            for tag in additional_tags:
+                try:
+                    print(f"DEBUG: Pushing additional tag: {tag}")
+
+                    # Push using requests
+                    push_headers = headers.copy()
+                    push_headers.update({
+                        "Content-Type": content_type
+                    })
+
+                    # Create push URL for the new tag
+                    tag_url = f"{registry_url}/v2/{repo_name}/manifests/{tag}"
+
+                    # Push the manifest using requests
+                    tag_resp = requests.put(tag_url, data=manifest_content, headers=push_headers, verify=verify)
+
+                    if tag_resp.status_code in [200, 201]:
+                        print(f"DEBUG: Successfully pushed tag {tag}")
+                        logger.info(f"Successfully pushed tag {tag}")
+                    else:
+                        print(f"DEBUG: Failed to push tag {tag}: {tag_resp.status_code}")
+                        logger.error(f"Failed to push tag {tag}: {tag_resp.status_code}")
+
+                except Exception as e:
+                    print(f"DEBUG: Error pushing tag {tag}: {str(e)}")
+                    logger.error(f"Error pushing tag {tag}: {str(e)}")
+
+        except Exception as e:
+            print(f"DEBUG: Error in push_additional_tags: {str(e)}")
+            logger.error(f"Error in push_additional_tags: {str(e)}")
 
 def extract_tar(tar: str, tmpdir: str):
     """