Merge pull request #52 from gardenlinux/48-do-we-have-to-include-this-applicationvndgardenlinuximagechecksumsha256

Removal of additional sha256 digest in manifest elements

Author: Malte Münch

src/python_gardenlinux_lib/features/parse_features.py

@@ -7,6 +7,36 @@
 import subprocess
 from typing import Optional
 
+# It is important that this list is sorted in descending length of the entries
+GL_MEDIA_TYPES = [
+    "gcpimage.tar.gz.log",
+    "firecracker.tar.gz",
+    "gcpimage.tar.gz",
+    "pxe.tar.gz.log",
+    "manifest.log",
+    "release.log",
+    "pxe.tar.gz",
+    "qcow2.log",
+    "test-log",
+    "manifest",
+    "vmdk.log",
+    "tar.log",
+    "release",
+    "vhd.log",
+    "ova.log",
+    "raw.log",
+    "tar.gz",
+    "qcow2",
+    "tar",
+    "iso",
+    "oci",
+    "vhd",
+    "vmdk",
+    "ova",
+    "raw",
+]
+
+
 GL_MEDIA_TYPE_LOOKUP = {
     "tar": "application/io.gardenlinux.image.archive.format.tar",
     "tar.gz": "application/io.gardenlinux.image.archive.format.tar.gz",
@@ -20,6 +50,19 @@
     "vmdk": "application/io.gardenlinux.image.format.vmdk",
     "ova": "application/io.gardenlinux.image.format.ova",
     "raw": "application/io.gardenlinux.image.archive.format.raw",
+    "manifest.log": "application/io.gardenlinux.log",
+    "release.log": "application/io.gardenlinux.log",
+    "test-log": "application/io.gardenlinux.test-log",
+    "manifest": "application/io.gardenlinux.manifest",
+    "tar.log": "application/io.gardenlinux.log",
+    "release": "application/io.gardenlinux.release",
+    "raw.log": "application/io.gardenlinux.log",
+    "qcow2.log": "application/io.gardenlinux.log",
+    "pxe.tar.gz.log": "application/io.gardenlinux.log",
+    "gcpimage.tar.gz.log": "application/io.gardenlinux.log",
+    "vmdk.log": "application/io.gardenlinux.log",
+    "vhd.log": "application/io.gardenlinux.log",
+    "ova.log": "application/io.gardenlinux.log",
 }
 
 
@@ -88,53 +131,82 @@ def construct_layer_metadata(
     return {
         "file_name": f"{cname}-{arch}-{version}-{commit}.{filetype}",
         "media_type": media_type,
+        "annotations": {"io.gardenlinux.image.layer.architecture": arch},
+    }
+
+
+def construct_layer_metadata_from_filename(filename: str, arch: str) -> dict:
+    """
+    :param str filename: filename of the blob
+    :param str arch: the arch of the target image
+    :return: dict of oci layer metadata for a given layer file
+    """
+    media_type = lookup_media_type_for_file(filename)
+    return {
+        "file_name": filename,
+        "media_type": media_type,
+        "annotations": {"io.gardenlinux.image.layer.architecture": arch},
     }
 
 
-def get_oci_metadata(cname: str, version: str, gardenlinux_root: str):
+def get_file_set_from_cname(cname: str, version: str, arch: str, gardenlinux_root: str):
     """
     :param str cname: the target cname of the image
-    :param str version: the target version of the image
+    :param str version: the version of the target image
+    :param str arch: the arch of the target image
     :param str gardenlinux_root: path of garden linux src root
-    :return: list of dicts, where each dict represents a layer
+    :return: set of file names for a given cname
     """
-    oci_layer_metadata_list = list()
+    file_set = set()
     features_by_type = get_features_dict(cname, gardenlinux_root)
     commit_str = get_gardenlinux_commit(gardenlinux_root, 8)
 
     if commit_str == "local":
         raise ValueError("Using local commit. Refusing to upload to OCI Registry")
-
-    for arch in ["amd64", "arm64"]:
-        for platform in features_by_type["platform"]:
-            image_file_types = deduce_image_filetype(
-                f"{gardenlinux_root}/features/{platform}"
-            )
-            archive_file_types = deduce_archive_filetype(
-                f"{gardenlinux_root}/features/{platform}"
+    for platform in features_by_type["platform"]:
+        image_file_types = deduce_filetypes(f"{gardenlinux_root}/features/{platform}")
+        for ft in image_file_types:
+            file_set.add(
+                f"{cname}-{arch}-{version}-{commit_str}.{ft}",
             )
-            # Allow multiple image scripts per feature
-            if not image_file_types:
-                image_file_types.append("raw")
-            if not archive_file_types:
-                image_file_types.append("tar")
-            for ft in archive_file_types:
-                cur_layer_metadata = construct_layer_metadata(
-                    ft, cname, version, arch, commit_str
-                )
-                cur_layer_metadata["annotations"] = {
-                    "io.gardenlinux.image.layer.architecture": arch
-                }
-                oci_layer_metadata_list.append(cur_layer_metadata)
-            # Allow multiple convert scripts per feature
-            for ft in image_file_types:
-                cur_layer_metadata = construct_layer_metadata(
-                    ft, cname, version, arch, commit_str
-                )
-                cur_layer_metadata["annotations"] = {
-                    "io.gardenlinux.image.layer.architecture": arch
-                }
-                oci_layer_metadata_list.append(cur_layer_metadata)
+    return file_set
+
+
+def get_oci_metadata_from_fileset(fileset: set, arch: str):
+    """
+    :param str arch: arch of the target image
+    :param set fileset: a list of filenames (not paths) to set oci_metadata for
+    :return: list of dicts, where each dict represents a layer
+    """
+    oci_layer_metadata_list = list()
+
+    for file in fileset:
+        oci_layer_metadata_list.append(
+            construct_layer_metadata_from_filename(file, arch)
+        )
+
+    return oci_layer_metadata_list
+
+
+def get_oci_metadata(cname: str, version: str, arch: str, gardenlinux_root: str):
+    """
+    :param str cname: the target cname of the image
+    :param str version: the target version of the image
+    :param str arch: arch of the target image
+    :param str gardenlinux_root: path of garden linux src root
+    :return: list of dicts, where each dict represents a layer
+    """
+
+    # This is the feature deduction approach (glcli oci push)
+    file_set = get_file_set_from_cname(cname, version, arch, gardenlinux_root)
+
+    # This is the tarball extraction approach (glcli oci push-tarball)
+    oci_layer_metadata_list = list()
+
+    for file in file_set:
+        oci_layer_metadata_list.append(
+            construct_layer_metadata_from_filename(file, arch)
+        )
 
     return oci_layer_metadata_list
 
@@ -148,7 +220,21 @@ def lookup_media_type_for_filetype(filetype: str) -> str:
         return GL_MEDIA_TYPE_LOOKUP[filetype]
     else:
         raise ValueError(
-            f"No media type for {filetype} is defined. You may want to add the definition to parse_features_lib"
+            f"media type for {filetype} is not defined. You may want to add the definition to parse_features_lib"
+        )
+
+
+def lookup_media_type_for_file(filename: str) -> str:
+    """
+    :param str filename: filename of the target layer
+    :return: mediatype
+    """
+    for suffix in GL_MEDIA_TYPES:
+        if filename.endswith(suffix):
+            return GL_MEDIA_TYPE_LOOKUP[suffix]
+    else:
+        raise ValueError(
+            f"media type for {filename} is not defined. You may want to add the definition to parse_features_lib"
         )
 
 
@@ -163,25 +249,40 @@ def deduce_feature_name(feature_dir: str):
     return parsed["name"]
 
 
-def deduce_archive_filetype(feature_dir):
+def deduce_archive_filetypes(feature_dir):
+    """
+    :param str feature_dir: Directory of single Feature
+    :return: str list of filetype for archive
+    """
+    return deduce_filetypes_from_string(feature_dir, "image")
+
+
+def deduce_image_filetypes(feature_dir):
     """
     :param str feature_dir: Directory of single Feature
-    :return: str of filetype for archive
+    :return: str list of filetype for image
     """
-    return deduce_filetype_from_string(feature_dir, "image")
+    return deduce_filetypes_from_string(feature_dir, "convert")
 
 
-def deduce_image_filetype(feature_dir):
+def deduce_filetypes(feature_dir):
     """
     :param str feature_dir: Directory of single Feature
-    :return: str of filetype for image
+    :return: str list of filetypes for the feature
     """
-    return deduce_filetype_from_string(feature_dir, "convert")
+    image_file_types = deduce_image_filetypes(feature_dir)
+    archive_file_types = deduce_archive_filetypes(feature_dir)
+    if not image_file_types:
+        image_file_types.append("raw")
+    if not archive_file_types:
+        archive_file_types.append("tar")
+    image_file_types.extend(archive_file_types)
+    return image_file_types
 
 
-def deduce_filetype_from_string(feature_dir: str, script_base_name: str):
+def deduce_filetypes_from_string(feature_dir: str, script_base_name: str):
     """
-    Garden Linux features can optionally have a image.<filetype> or convert.<filetype> script,
+    Garden Linux features can optionally have an image.<filetype> or convert.<filetype> script,
     where the <filetype> indicates the target filetype.
 
     image.<filetype> script converts the .tar to <filetype> archive.
@@ -229,7 +330,7 @@ def read_feature_files(feature_dir):
                     )
                 feature_graph.add_edge(node, ref, attr=attr)
     if not networkx.is_directed_acyclic_graph(feature_graph):
-        raise ValueError("Graph is not directed asyclic graph")
+        raise ValueError("Graph is not directed acyclic graph")
     return feature_graph
 
 

src/python_gardenlinux_lib/oras/registry.py

@@ -74,13 +74,13 @@ def NewPlatform(architecture: str, version: str) -> dict:
 
 
 def NewManifestMetadata(
-    digest: str, size: int, annotaions: dict, platform_data: dict
+    digest: str, size: int, annotations: dict, platform_data: dict
 ) -> dict:
     manifest_meta_data = copy.deepcopy(EmptyManifestMetadata)
     manifest_meta_data["mediaType"] = "application/vnd.oci.image.manifest.v1+json"
     manifest_meta_data["digest"] = digest
     manifest_meta_data["size"] = size
-    manifest_meta_data["annotations"] = annotaions
+    manifest_meta_data["annotations"] = annotations
     manifest_meta_data["platform"] = platform_data
     manifest_meta_data["artifactType"] = ""
     return manifest_meta_data
@@ -114,7 +114,10 @@ def create_config_from_dict(conf: dict, annotations: dict) -> Tuple[dict, str]:
 def construct_manifest_entry_signed_data_string(
     cname: str, version: str, new_manifest_metadata: dict, architecture: str
 ) -> str:
-    data_to_sign = f"versio:{version}  cname{cname}  architecture:{architecture}  manifest-size:{new_manifest_metadata['size']}  manifest-digest:{new_manifest_metadata['digest']}"
+    data_to_sign = (
+        f"version:{version}  cname:{cname}  architecture:{architecture}  manifest-size"
+        f":{new_manifest_metadata['size']}  manifest-digest:{new_manifest_metadata['digest']}"
+    )
     return data_to_sign
 
 
@@ -494,9 +497,7 @@ def verify_manifest_signature(self, manifest: dict):
             if annotation_signed_string_key not in layer["annotations"]:
                 raise ValueError(f"layer is not signed. layer: {layer}")
             media_type = layer["mediaType"]
-            checksum_sha256 = layer["annotations"][
-                "application/vnd.gardenlinux.image.checksum.sha256"
-            ]
+            checksum_sha256 = layer["digest"].removeprefix("sha256:")
             signature = layer["annotations"][annotation_signature_key]
             signed_data = layer["annotations"][annotation_signed_string_key]
             signed_data_expected = construct_layer_signed_data_string(
@@ -559,20 +560,21 @@ def push_image_manifest(
         architecture: str,
         cname: str,
         version: str,
-        gardenlinux_root: str,
         build_artifacts_dir: str,
+        oci_metadata: list,
     ):
         """
         creates and pushes an image manifest
 
+        :param oci_metadata: a list of filenames and their OCI metadata, can be constructed with get_oci_metadata
         :param str architecture: target architecture of the image
         :param str cname: canonical name of the target image
         :param str build_artifacts_dir: directory where the build artifacts are located
         """
 
         # TODO: construct oci_artifacts default data
 
-        oci_metadata = get_oci_metadata(cname, version, gardenlinux_root)
+        # oci_metadata = get_oci_metadata(cname, version, architecture, gardenlinux_root)
 
         manifest_image = oras.oci.NewManifest()
         total_size = 0
@@ -690,8 +692,7 @@ def create_layer(
         checksum_sha256 = calculate_sha256(file_path)
         layer = oras.oci.NewLayer(file_path, media_type, is_dir=False)
         layer["annotations"] = {
-            oras.defaults.annotation_title: file_path,
-            "application/vnd.gardenlinux.image.checksum.sha256": checksum_sha256,
+            oras.defaults.annotation_title: os.path.basename(file_path),
         }
         self.sign_layer(
             layer, cname, version, architecture, checksum_sha256, media_type

tests/test_deduce_image_type.py

@@ -1,6 +1,6 @@
 from python_gardenlinux_lib.features.parse_features import (
-    deduce_archive_filetype,
-    deduce_image_filetype,
+    deduce_archive_filetypes,
+    deduce_image_filetypes,
 )
 import pytest
 
@@ -20,7 +20,7 @@
     ],
 )
 def test_deduce_image_type(feature_name, expected_file_type):
-    file_type = deduce_image_filetype(f"{GL_ROOT_DIR}/features/{feature_name}")
+    file_type = deduce_image_filetypes(f"{GL_ROOT_DIR}/features/{feature_name}")
     assert sorted(expected_file_type) == file_type
 
 
@@ -35,5 +35,5 @@ def test_deduce_image_type(feature_name, expected_file_type):
     ],
 )
 def test_deduce_archive_type(feature_name, expected_file_type):
-    file_type = deduce_archive_filetype(f"{GL_ROOT_DIR}/features/{feature_name}")
+    file_type = deduce_archive_filetypes(f"{GL_ROOT_DIR}/features/{feature_name}")
     assert sorted(expected_file_type) == file_type