diff --git a/.github/actions/features_parse/action.yml b/.github/actions/features_parse/action.yml index dd55a841..ac7e7bba 100644 --- a/.github/actions/features_parse/action.yml +++ b/.github/actions/features_parse/action.yml @@ -11,7 +11,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.7.0 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.7.2 - id: result shell: bash run: | diff --git a/.github/actions/flavors_parse/action.yml b/.github/actions/flavors_parse/action.yml index dcc00bfb..90576d05 100644 --- a/.github/actions/flavors_parse/action.yml +++ b/.github/actions/flavors_parse/action.yml @@ -13,7 +13,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.7.0 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.7.2 - id: matrix shell: bash run: | diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 2e7eb29e..24138507 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -3,7 +3,7 @@ description: Installs the given GardenLinux Python library inputs: version: description: GardenLinux Python library version - default: "0.7.0" + default: "0.7.2" runs: using: composite steps: diff --git a/pyproject.toml b/pyproject.toml index b6d4d19c..9fa2c959 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gardenlinux" -version = "0.7.0" +version = "0.7.2" description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames" authors = ["Garden Linux Maintainers "] license = "Apache-2.0" diff --git a/src/gardenlinux/oci/container.py b/src/gardenlinux/oci/container.py index 392769e0..ea2965c4 100644 --- a/src/gardenlinux/oci/container.py +++ b/src/gardenlinux/oci/container.py @@ -39,7 +39,7 @@ class Container(Registry): :author: Garden Linux Maintainers :copyright: Copyright 2024 SAP SE :package: gardenlinux - :subpackage: flavors + :subpackage: oci :since: 0.7.0 :license: https://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 @@ -401,7 +401,7 @@ def push_manifest_and_artifacts( cleanup_blob = True manifest.append_layer(layer) - layer_dict = layer.to_dict() + layer_dict = layer.dict self._logger.debug(f"Layer: {layer_dict}") diff --git a/src/gardenlinux/oci/index.py b/src/gardenlinux/oci/index.py index bd125981..62785588 100644 --- a/src/gardenlinux/oci/index.py +++ b/src/gardenlinux/oci/index.py @@ -7,7 +7,25 @@ class Index(dict): + """ + OCI image index + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: oci + :since: 0.7.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + def __init__(self, *args, **kwargs): + """ + Constructor __init__(Index) + + :since: 0.7.0 + """ + dict.__init__(self) self.update(deepcopy(EmptyIndex)) @@ -16,23 +34,45 @@ def __init__(self, *args, **kwargs): @property def json(self): + """ + Returns the OCI image index as a JSON + + :return: (bytes) OCI image index as JSON + :since: 0.7.0 + """ + return json.dumps(self).encode("utf-8") @property def manifests_as_dict(self): + """ + Returns the OCI image manifests of the index + + :return: (dict) OCI image manifests with CNAME or digest as key + :since: 0.7.0 + """ + manifests = {} for manifest in self["manifests"]: - if "cname" not in manifest.get("annotations", {}): - raise RuntimeError( - "Unexpected manifest with missing annotation 'cname' found" - ) + if "annotations" not in manifest or "cname" not in manifest["annotations"]: + manifest_key = manifest["digest"] + else: + manifest_key = manifest["annotations"]["cname"] - manifests[manifest["annotations"]["cname"]] = manifest + manifests[manifest_key] = manifest return manifests def append_manifest(self, manifest): + """ + Appends the given OCI image manifest to the index + + :param manifest: OCI image manifest + + :since: 0.7.0 + """ + if not isinstance(manifest, dict): raise RuntimeError("Unexpected manifest type given") diff --git a/src/gardenlinux/oci/layer.py b/src/gardenlinux/oci/layer.py index 23b24fe0..6c4612e7 100644 --- a/src/gardenlinux/oci/layer.py +++ b/src/gardenlinux/oci/layer.py @@ -13,12 +13,34 @@ class Layer(_Layer, Mapping): + """ + OCI image layer + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: oci + :since: 0.7.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + def __init__( self, blob_path: PathLike | str, media_type: Optional[str] = None, is_dir: bool = False, ): + """ + Constructor __init__(Index) + + :param blob_path: The path of the blob for the layer + :param media_type: Media type for the blob (optional) + :param is_dir: Is the blob a directory? + + :since: 0.7.0 + """ + if not isinstance(blob_path, PathLike): blob_path = Path(blob_path) @@ -28,11 +50,26 @@ def __init__( ANNOTATION_TITLE: blob_path.name, } + @property + def dict(self): + """ + Return a dictionary representation of the layer + + :return: (dict) OCI manifest layer metadata dictionary + :since: 0.7.2 + """ + layer = _Layer.to_dict(self) + layer["annotations"] = self._annotations + + return layer + def __delitem__(self, key): """ python.org: Called to implement deletion of self[key]. :param key: Mapping key + + :since: 0.7.0 """ if key == "annotations": @@ -49,6 +86,7 @@ def __getitem__(self, key): :param key: Mapping key :return: (mixed) Mapping key value + :since: 0.7.0 """ if key == "annotations": @@ -63,6 +101,7 @@ def __iter__(self): python.org: Return an iterator object. :return: (object) Iterator object + :since: 0.7.0 """ iter(_SUPPORTED_MAPPING_KEYS) @@ -71,7 +110,8 @@ def __len__(self): """ python.org: Called to implement the built-in function len(). - :return: (int) Number of database instance attributes + :return: (int) Number of attributes + :since: 0.7.0 """ return len(_SUPPORTED_MAPPING_KEYS) @@ -82,6 +122,8 @@ def __setitem__(self, key, value): :param key: Mapping key :param value: self[key] value + + :since: 0.7.0 """ if key == "annotations": @@ -91,21 +133,16 @@ def __setitem__(self, key, value): f"'{self.__class__.__name__}' object is not subscriptable except for keys: {_SUPPORTED_MAPPING_KEYS}" ) - def to_dict(self): - """ - Return a dictionary representation of the layer - """ - layer = _Layer.to_dict(self) - layer["annotations"] = self._annotations - - return layer - @staticmethod def generate_metadata_from_file_name(file_name: PathLike | str, arch: str) -> dict: """ - :param str file_name: file_name of the blob - :param str arch: the arch of the target image - :return: dict of oci layer metadata for a given layer file + Generates OCI manifest layer metadata for the given file path and name. + + :param file_name: File path and name of the target layer + :param arch: The arch of the target image + + :return: (dict) OCI manifest layer metadata dictionary + :since: 0.7.0 """ if not isinstance(file_name, PathLike): @@ -122,8 +159,12 @@ def generate_metadata_from_file_name(file_name: PathLike | str, arch: str) -> di @staticmethod def lookup_media_type_for_file_name(file_name: str) -> str: """ - :param str file_name: file_name of the target layer - :return: mediatype + Looks up the media type based on file extension. + + :param file_name: File path and name of the target layer + + :return: (str) Media type + :since: 0.7.0 """ if not isinstance(file_name, PathLike): diff --git a/src/gardenlinux/oci/manifest.py b/src/gardenlinux/oci/manifest.py index 438172a6..6ac7f2b5 100644 --- a/src/gardenlinux/oci/manifest.py +++ b/src/gardenlinux/oci/manifest.py @@ -15,7 +15,25 @@ class Manifest(dict): + """ + OCI image manifest + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: oci + :since: 0.7.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + def __init__(self, *args, **kwargs): + """ + Constructor __init__(Manifest) + + :since: 0.7.0 + """ + dict.__init__(self) self._config_bytes = b"{}" @@ -26,6 +44,13 @@ def __init__(self, *args, **kwargs): @property def arch(self): + """ + Returns the architecture of the OCI image manifest. + + :return: (str) OCI image architecture + :since: 0.7.0 + """ + if "architecture" not in self.get("annotations", {}): raise RuntimeError( "Unexpected manifest with missing config annotation 'architecture' found" @@ -35,11 +60,26 @@ def arch(self): @arch.setter def arch(self, value): + """ + Sets the architecture of the OCI image manifest. + + :param value: OCI image architecture + + :since: 0.7.0 + """ + self._ensure_annotations_dict() self["annotations"]["architecture"] = value @property def cname(self): + """ + Returns the GardenLinux canonical name of the OCI image manifest. + + :return: (str) OCI image GardenLinux canonical name + :since: 0.7.0 + """ + if "cname" not in self.get("annotations", {}): raise RuntimeError( "Unexpected manifest with missing config annotation 'cname' found" @@ -49,11 +89,26 @@ def cname(self): @cname.setter def cname(self, value): + """ + Sets the GardenLinux canonical name of the OCI image manifest. + + :param value: OCI image GardenLinux canonical name + + :since: 0.7.0 + """ + self._ensure_annotations_dict() self["annotations"]["cname"] = value @property def commit(self): + """ + Returns the GardenLinux Git commit ID of the OCI image manifest. + + :return: (str) OCI image GardenLinux Git commit ID + :since: 0.7.0 + """ + if "commit" not in self.get("annotations", {}): raise RuntimeError( "Unexpected manifest with missing config annotation 'commit' found" @@ -63,20 +118,49 @@ def commit(self): @commit.setter def commit(self, value): + """ + Sets the GardenLinux Git commit ID of the OCI image manifest. + + :param value: OCI image GardenLinux Git commit ID + + :since: 0.7.0 + """ + self._ensure_annotations_dict() self["annotations"]["commit"] = value @property def config_json(self): + """ + Returns the OCI image manifest config. + + :return: (bytes) OCI image manifest config + :since: 0.7.0 + """ + return self._config_bytes @property def digest(self): + """ + Returns the OCI image manifest digest. + + :return: (str) OCI image manifest digest + :since: 0.7.0 + """ + digest = sha256(self.json).hexdigest() return f"sha256:{digest}" @property def feature_set(self): + """ + Returns the GardenLinux feature set of the OCI image manifest. + + :return: (str) OCI image GardenLinux feature set + :since: 0.7.0 + """ + if "feature_set" not in self.get("annotations", {}): raise RuntimeError( "Unexpected manifest with missing config annotation 'feature_set' found" @@ -86,19 +170,48 @@ def feature_set(self): @feature_set.setter def feature_set(self, value): + """ + Sets the GardenLinux feature set of the OCI image manifest. + + :param value: OCI image GardenLinux feature set + + :since: 0.7.0 + """ + self._ensure_annotations_dict() self["annotations"]["feature_set"] = value @property def flavor(self): + """ + Returns the GardenLinux flavor of the OCI image manifest. + + :return: (str) OCI image GardenLinux flavor + :since: 0.7.0 + """ + return CName(self.cname).flavor @property def json(self): + """ + Returns the OCI image manifest as a JSON + + :return: (bytes) OCI image manifest as JSON + :since: 0.7.0 + """ + return json.dumps(self).encode("utf-8") @property def layers_as_dict(self): + """ + Returns the OCI image manifest layers as a dictionary. + + :return: (dict) OCI image manifest layers with title as key + :since: 0.7.0 + """ + layers = {} for layer in self["layers"]: @@ -113,10 +226,24 @@ def layers_as_dict(self): @property def size(self): + """ + Returns the OCI image manifest JSON size in bytes. + + :return: (int) OCI image manifest JSON size in bytes + :since: 0.7.0 + """ + return len(self.json) @property def version(self): + """ + Returns the GardenLinux version of the OCI image manifest. + + :return: (str) OCI image GardenLinux version + :since: 0.7.0 + """ + if "version" not in self.get("annotations", {}): raise RuntimeError( "Unexpected manifest with missing config annotation 'version' found" @@ -126,12 +253,21 @@ def version(self): @version.setter def version(self, value): + """ + Sets the GardenLinux version of the OCI image manifest. + + :param value: OCI image GardenLinux version + + :since: 0.7.0 + """ + self._ensure_annotations_dict() self["annotations"]["version"] = value def config_from_dict(self, config: dict, annotations: dict): """ - Write a new OCI configuration to file, and generate oci metadata for it + Write a new OCI configuration to file, and generate oci metadata for it. + For reference see https://github.com/opencontainers/image-spec/blob/main/config.md annotations, mediatype, size, digest are not part of digest and size calculation, and therefore must be attached to the output dict and not written to the file. @@ -139,6 +275,7 @@ def config_from_dict(self, config: dict, annotations: dict): :param config: dict with custom configuration (the payload of the configuration) :param annotations: dict with custom annotations to be attached to metadata part of config + :since: 0.7.0 """ self._config_bytes = json.dumps(config).encode("utf-8") @@ -151,10 +288,18 @@ def config_from_dict(self, config: dict, annotations: dict): self["config"] = config def append_layer(self, layer): + """ + Appends the given OCI image manifest layer to the manifest + + :param layer: OCI image manifest layer + + :since: 0.7.0 + """ + if not isinstance(layer, Layer): raise RuntimeError("Unexpected layer type given") - layer_dict = layer.to_dict() + layer_dict = layer.dict if "org.opencontainers.image.title" not in layer_dict.get("annotations", {}): raise RuntimeError( diff --git a/src/gardenlinux/oci/wrapper.py b/src/gardenlinux/oci/wrapper.py deleted file mode 100644 index d531d2e8..00000000 --- a/src/gardenlinux/oci/wrapper.py +++ /dev/null @@ -1,87 +0,0 @@ -import functools -import logging -import sys -import time - -from requests.exceptions import RequestException, HTTPError - -logger = logging.getLogger(__name__) -logging.basicConfig(stream=sys.stdout, level=logging.INFO) - - -def retry_on_error( - max_retries=3, - initial_delay=1, - backoff_factor=2, - # retry on: - # - 502 - bad gateway - # - 503 - service unavailable - # - 504 - gateway timeout - # - 429 - too many requests - # - 400 - bad request (e.g. invalid range start for blob upload) - # - 404 - not found (e.g. blob not found is seen in unit tests) - retryable_status_codes=(502, 503, 504, 429, 400, 404), - retryable_exceptions=(RequestException,), -): - """ - A decorator for retrying functions that might fail due to transient network issues. - - Args: - max_retries: Maximum number of retry attempts - initial_delay: Initial delay between retries in seconds - backoff_factor: Factor by which the delay increases with each retry - retryable_status_codes: HTTP status codes that trigger a retry - retryable_exceptions: Exception types that trigger a retry - - Returns: - Decorated function with retry logic - """ - - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - delay = initial_delay - last_exception = None - - for retry_count in range(max_retries + 1): - try: - if retry_count > 0: - logger.info( - f"Retry attempt {retry_count}/{max_retries} for {func.__name__}" - ) - - response = func(*args, **kwargs) - - # Check for retryable status codes in the response - if ( - hasattr(response, "status_code") - and response.status_code in retryable_status_codes - ): - status_code = response.status_code - logger.warning( - f"Received status code {status_code} from {func.__name__}, retrying..." - ) - last_exception = HTTPError(f"HTTP Error {status_code}") - else: - # Success, return the response - return response - - except retryable_exceptions as e: - logger.warning(f"Request failed in {func.__name__}: {str(e)}") - last_exception = e - - # Don't sleep if this was the last attempt - if retry_count < max_retries: - sleep_time = delay * (backoff_factor**retry_count) - logger.info(f"Waiting {sleep_time:.2f} seconds before retry") - time.sleep(sleep_time) - - # If we got here, all retries failed - logger.error(f"All {max_retries} retries failed for {func.__name__}") - if last_exception: - raise last_exception - return None - - return wrapper - - return decorator