Merge pull request #33 from gardenlinux/no-yaml-required

oci push can be used without any modifications to gardenlinux features. This commit adds feature to automatically deduce the build artifacts from the cname.

THe pytests automatically create dummy artifacts, so we do not need to re-build gardenlinux in all flavors to test this library.

Author: Vincinator

.github/workflows/pytests.yml

@@ -13,6 +13,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          submodules: 'true'
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.gitignore

@@ -1,4 +1,3 @@
-gardenlinux
 _build
 
 # Byte-compiled / optimized / DLL files

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test-data/gardenlinux"]
+	path = test-data/gardenlinux
+	url = https://github.com/gardenlinux/gardenlinux

cert/gencert.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERT_NAME="$SCRIPT_DIR/oci-sign"
+
+
+openssl req -x509 -newkey rsa -pkeyopt rsa_keygen_bits:4096  -days 3650 -nodes -keyout $CERT_NAME.key -out $CERT_NAME.crt -subj "/CN=Garden Linux test signing key for oci"
+

poetry.lock

@@ -2,7 +2,7 @@
 name = "python_gardenlinux_lib"
 version = "0.1.0"
 description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames"
-authors = ["Malte Münch <[email protected]>"]
+authors = ["Garden Linux Maintainers <[email protected]>"]
 license = "Apache-2.0"
 readme = "README.md"
 packages = [{include = "python_gardenlinux_lib", from="src"}]
@@ -15,6 +15,11 @@ pytest = "^8.3.2"
 gitpython = "^3.1.43"
 sphinx-rtd-theme = "^2.0.0"
 apt-repo = "^0.5"
+jsonschema = "^4.23.0"
+oras = { git  = "https://github.com/oras-project/oras-py.git", rev="caf8db5b279382335fbb1f6d7402ed9b73618d37" }
+python-dotenv = "^1.0.1"
+cryptography = "^43.0.0"
+
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.8.0"
@@ -23,6 +28,7 @@ black = "^24.8.0"
 pythonpath = [
   "src"
 ]
+norecursedirs = "test-data"
 
 [build-system]
 requires = ["poetry-core"]

pyproject.toml

@@ -0,0 +1,81 @@
+from apt_repo import APTRepository
+from typing import Optional
+
+
+class GardenLinuxRepo(APTRepository):
+    def __init__(
+        self,
+        dist: str,
+        url: Optional[str] = "http://packages.gardenlinux.io/gardenlinux",
+        components: Optional[list[str]] = ["main"],
+    ) -> None:
+        self.components = components
+        self.url = url
+        self.dist = dist
+        self.repo = APTRepository(self.url, self.dist, self.components)
+
+    def get_package_version_by_name(self, name: str) -> list[tuple[str, str]]:
+        """
+        :param str name: name of package to find
+        :returns: packages matching the input name
+        """
+        return [
+            (package.package, package.version)
+            for package in self.repo.get_packages_by_name(name)
+        ]
+
+    def get_packages_versions(self):
+        """
+        Returns list of (package, version) tuples
+        """
+        return [(p.package, p.version) for p in self.repo.packages]
+
+
+def compare_gardenlinux_repo_version(version_a: str, version_b: str):
+    """
+    :param str version_a: Version of first Garden Linux repo
+    :param str version_b: Version of first Garden Linux repo
+
+    Example: print(compare_gardenlinux_repo_version("1443.2", "1443.1"))
+    """
+    return compare_repo(GardenLinuxRepo(version_a), GardenLinuxRepo(version_b))
+
+
+def compare_repo(
+    a: GardenLinuxRepo, b: GardenLinuxRepo, available_in_both: Optional[bool] = False
+):
+    """
+    :param a GardenLinuxRepo: first repo to compare
+    :param b GardenLinuxRepo: second repo to compare
+    :returns: differences between repo a and repo b
+    """
+
+    packages_a = dict(a.get_packages_versions())
+    packages_b = dict(b.get_packages_versions())
+    if available_in_both:
+        all_names = set(packages_a.keys()).intersection(set(packages_b.keys()))
+    else:
+        all_names = set(packages_a.keys()).union(set(packages_b.keys()))
+
+    return [
+        (name, packages_a.get(name), packages_b.get(name))
+        for name in all_names
+        if (
+            name in packages_a
+            and name in packages_b
+            and packages_a[name] != packages_b[name]
+        )
+        or (name not in packages_b or name not in packages_a)
+    ]
+
+
+# EXAMPLE USAGE.
+# print(compare_gardenlinux_repo_version("1443.2", "1443.1"))
+
+# gl_repo = GardenLinuxRepo("today")
+# gl_repo_1592 = GardenLinuxRepo("1592.0")
+# deb_testing = GardenLinuxRepo("testing", "https://deb.debian.org/debian/")
+# print(compare_repo(gl_repo, gl_repo_1592, available_in_both=True))
+# print(compare_repo(gl_repo, deb_testing, available_in_both=True))
+# # print(gl_repo.get_packages_versions())
+# print(gl_repo.get_package_version_by_name("wget"))

src/python_gardenlinux_lib/features/__init__.py

@@ -9,6 +9,7 @@
 
 GL_MEDIA_TYPE_LOOKUP = {
     "tar": "application/io.gardenlinux.image.archive.format.tar",
+    "tar.gz": "application/io.gardenlinux.image.archive.format.tar.gz",
     "pxe.tar.gz": "application/io.gardenlinux.image.archive.format.pxe.tar.gz",
     "iso": "application/io.gardenlinux.image.archive.format.iso",
     "oci": "application/io.gardenlinux.image.archive.format.oci",
@@ -85,7 +86,7 @@ def construct_layer_metadata(
     """
     media_type = lookup_media_type_for_filetype(filetype)
     return {
-        "filename": f"{cname}-{version}-{arch}-{commit}.{filetype}",
+        "file_name": f"{cname}-{version}-{arch}-{commit}.{filetype}",
         "media_type": media_type,
     }
 

src/python_gardenlinux_lib/features/package_repo_info.py

@@ -8,7 +8,6 @@
 import uuid
 from enum import Enum, auto
 from typing import Optional, Tuple
-from python_gardenlinux_lib.parse_features import read_feature_files
 
 import jsonschema
 import oras.auth
@@ -17,8 +16,8 @@
 import oras.oci
 import oras.provider
 import oras.utils
+from python_gardenlinux_lib.features.parse_features import get_oci_metadata
 import requests
-import yaml
 from oras.container import Container as OrasContainer
 from oras.decorator import ensure_container
 from oras.provider import Registry
@@ -126,6 +125,29 @@ def construct_layer_signed_data_string(
     return data_to_sign
 
 
+def setup_registry(
+    container_name: str,
+    private_key: Optional[str] = None,
+    insecure: bool = False,
+    public_key: Optional[str] = None,
+):
+    username = os.getenv("GLOCI_REGISTRY_USERNAME")
+    token = os.getenv("GLOCI_REGISTRY_TOKEN")
+    if username is None:
+        logger.error("No username")
+        raise ValueError("No Username provided when setting up registry")
+    if token is None:
+        logger.error("No token")
+        raise ValueError("No token provided when setting up registry")
+    return GlociRegistry(
+        container_name,
+        token,
+        insecure=insecure,
+        private_key=private_key,
+        public_key=public_key,
+    )
+
+
 class GlociRegistry(Registry):
     def __init__(
         self,
@@ -533,27 +555,35 @@ def upload_index(self, index: dict) -> requests.Response:
         return response
 
     def push_image_manifest(
-        self, architecture: str, cname: str, version: str, info_yaml: str
+        self,
+        architecture: str,
+        cname: str,
+        version: str,
+        gardenlinux_root: str,
+        build_artifacts_dir: str,
     ):
         """
         creates and pushes an image manifest
+
+        :param str architecture: target architecture of the image
+        :param str cname: canonical name of the target image
+        :param str build_artifacts_dir: directory where the build artifacts are located
         """
-        # container = OrasContainer(container_name)
-        with open(info_yaml, "r") as f:
-            info_data = yaml.safe_load(f)
-            base_path = os.path.join(os.path.dirname(info_yaml))
+
+        # TODO: construct oci_artifacts default data
+
+        oci_metadata = get_oci_metadata(cname, version, gardenlinux_root)
 
         manifest_image = oras.oci.NewManifest()
         total_size = 0
 
-        for artifact in info_data["oci_artifacts"]:
+        for artifact in oci_metadata:
             annotations_input = artifact["annotations"]
             media_type = artifact["media_type"]
-            file_path = os.path.join(base_path, artifact["file_name"])
+            file_path = os.path.join(build_artifacts_dir, artifact["file_name"])
 
             if not os.path.exists(file_path):
-                logger.error(f"{file_path} does not exist.")
-                continue
+                raise ValueError(f"{file_path} does not exist.")
 
             cleanup_blob = False
             if os.path.isdir(file_path):
@@ -573,25 +603,26 @@ def push_image_manifest(
             self._check_200_response(response)
             if cleanup_blob and os.path.exists(file_path):
                 os.remove(file_path)
-        layer = self.create_layer(
-            info_yaml,
-            cname,
-            version,
-            architecture,
-            "application/io.gardenlinux.oci.info-yaml",
-        )
-        total_size += int(layer["size"])
-        manifest_image["layers"].append(layer)
+        # layer = self.create_layer(
+        #     info_yaml,
+        #     cname,
+        #     version,
+        #     architecture,
+        #     "application/io.gardenlinux.oci.info-yaml",
+        # )
+        # total_size += int(layer["size"])
+        # manifest_image["layers"].append(layer)
+
         manifest_image["annotations"] = {}
         manifest_image["annotations"]["version"] = version
         manifest_image["annotations"]["cname"] = cname
         manifest_image["annotations"]["architecture"] = architecture
         attach_state(manifest_image["annotations"], "UNTESTED")
 
-        if layer is None:
-            raise ValueError("error: layer is none")
-        response = self.upload_blob(info_yaml, self.container, layer)
-        self._check_200_response(response)
+        # if layer is None:
+        #    raise ValueError("error: layer is none")
+        # response = self.upload_blob(info_yaml, self.container, layer)
+        # self._check_200_response(response)
 
         config_annotations = {"cname": cname, "architecture": architecture}
         conf, config_file = create_config_from_dict(dict(), config_annotations)