diff --git a/.github/actions/features_parse/action.yml b/.github/actions/features_parse/action.yml index 7ee5d3ff..d415b96b 100644 --- a/.github/actions/features_parse/action.yml +++ b/.github/actions/features_parse/action.yml @@ -11,7 +11,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.7.3 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.0 - id: result shell: bash run: | diff --git a/.github/actions/flavors_parse/action.yml b/.github/actions/flavors_parse/action.yml index ddce442c..8864ba41 100644 --- a/.github/actions/flavors_parse/action.yml +++ b/.github/actions/flavors_parse/action.yml @@ -13,7 +13,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.7.3 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.0 - id: matrix shell: bash run: | diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 677bbffb..7797d4fc 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -3,7 +3,7 @@ description: Installs the given GardenLinux Python library inputs: version: description: GardenLinux Python library version - default: "0.7.3" + default: "0.8.0" runs: using: composite steps: diff --git a/pyproject.toml b/pyproject.toml index 84e5e9da..370eb01f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gardenlinux" -version = "0.7.3" +version = "0.8.0" description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames" authors = ["Garden Linux Maintainers "] license = "Apache-2.0" @@ -34,6 +34,7 @@ gl-cname = "gardenlinux.features.cname_main:main" gl-features-parse = "gardenlinux.features.__main__:main" gl-flavors-parse = "gardenlinux.flavors.__main__:main" gl-oci = "gardenlinux.oci.__main__:main" +gl-s3 = "gardenlinux.s3.__main__:main" [tool.pytest.ini_options] pythonpath = ["src"] diff --git a/src/gardenlinux/features/cname.py b/src/gardenlinux/features/cname.py index 9085da21..fcd90265 100644 --- a/src/gardenlinux/features/cname.py +++ b/src/gardenlinux/features/cname.py @@ -128,6 +128,16 @@ def feature_set(self) -> str: return Parser().filter_as_string(self.flavor) + @property + def platform(self) -> str: + """ + Returns the platform for the cname parsed. + + :return: (str) Flavor + """ + + return re.split("[_-]", self._flavor, 1)[0] + @property def version(self) -> Optional[str]: """ diff --git a/src/gardenlinux/s3/__init__.py b/src/gardenlinux/s3/__init__.py new file mode 100644 index 00000000..ec76b3b7 --- /dev/null +++ b/src/gardenlinux/s3/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +""" +S3 module +""" + +from .bucket import Bucket +from .s3_artifacts import S3Artifacts + +__all__ = ["Bucket", "S3Artifacts"] diff --git a/src/gardenlinux/s3/__main__.py b/src/gardenlinux/s3/__main__.py new file mode 100644 index 00000000..6e46c019 --- /dev/null +++ b/src/gardenlinux/s3/__main__.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +gl-s3 main entrypoint +""" + +import argparse +import os +import re +import sys +from functools import reduce +from pathlib import Path +from typing import Any, List, Set + +from .s3_artifacts import S3Artifacts + + +_ARGS_ACTION_ALLOWED = [ + "download-artifacts-from-bucket", + "upload-artifacts-to-bucket", +] + + +def main() -> None: + """ + gl-s3 main() + + :since: 0.8.0 + """ + + parser = argparse.ArgumentParser() + + parser.add_argument("--bucket", dest="bucket") + parser.add_argument("--cname", required=False, dest="cname") + parser.add_argument("--path", required=False, dest="path") + + parser.add_argument("action", nargs="?", choices=_ARGS_ACTION_ALLOWED) + + args = parser.parse_args() + + if args.action == "download-artifacts-from-bucket": + S3Artifacts(args.bucket).download_to_directory(args.cname, args.path) + elif args.action == "upload-artifacts-to-bucket": + S3Artifacts(args.bucket).upload_from_directory(args.cname, args.path) diff --git a/src/gardenlinux/s3/bucket.py b/src/gardenlinux/s3/bucket.py new file mode 100644 index 00000000..b9330e5f --- /dev/null +++ b/src/gardenlinux/s3/bucket.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +""" +S3 bucket +""" + +import boto3 +import logging +from typing import Any, Optional + +from ..logger import LoggerSetup + + +class Bucket(object): + """ + S3 bucket class + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: s3 + :since: 0.8.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + + def __init__( + self, + bucket_name: str, + endpoint_url: Optional[str] = None, + s3_resource_config: Optional[dict[str, Any]] = None, + logger: Optional[logging.Logger] = None, + ): + """ + Constructor __init__(Bucket) + + :param bucket_name: S3 bucket name + :param endpoint_url: S3 endpoint URL + :param s3_resource_config: Additional boto3 S3 config values + :param logger: Logger instance + + :since: 0.8.0 + """ + + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.s3") + + if s3_resource_config is None: + s3_resource_config = {} + + if endpoint_url is not None: + s3_resource_config["endpoint_url"] = endpoint_url + + self._s3_resource = boto3.resource("s3", **s3_resource_config) + + self._bucket = self._s3_resource.Bucket(bucket_name) + self._logger = logger + + @property + def objects(self): + """ + Returns a list of all objects in a bucket. + + :return: (list) S3 bucket objects + :since: 0.8.0 + """ + + return self._bucket.objects.all() + + def __getattr__(self, name): + """ + python.org: Called when an attribute lookup has not found the attribute in + the usual places (i.e. it is not an instance attribute nor is it found in the + class tree for self). + + :param name: Attribute name + + :return: (mixed) Attribute + :since: 0.8.0 + """ + + return getattr(self._bucket, name) diff --git a/src/gardenlinux/s3/s3_artifacts.py b/src/gardenlinux/s3/s3_artifacts.py new file mode 100644 index 00000000..e28bc912 --- /dev/null +++ b/src/gardenlinux/s3/s3_artifacts.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- + +""" +S3 GardenLinux artifacts +""" + +import logging +import yaml +from configparser import ConfigParser, UNNAMED_SECTION +from datetime import datetime +from hashlib import file_digest +from os import PathLike, stat +from os.path import basename +from pathlib import Path +from tempfile import TemporaryFile +from typing import Any, Optional +from urllib.parse import urlencode + +from ..features.cname import CName +from ..logger import LoggerSetup + +from .bucket import Bucket + + +class S3Artifacts(object): + """ + S3Artifacts support access to GardenLinux S3 resources. + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: s3 + :since: 0.8.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + + def __init__( + self, + bucket_name: str, + endpoint_url: Optional[str] = None, + s3_resource_config: Optional[dict[str, Any]] = None, + logger: Optional[logging.Logger] = None, + ): + """ + Constructor __init__(S3Artifacts) + + :param bucket_name: S3 bucket name + :param endpoint_url: S3 endpoint URL + :param s3_resource_config: Additional boto3 S3 config values + :param logger: Logger instance + + :since: 0.8.0 + """ + + self._bucket = Bucket(bucket_name, endpoint_url, s3_resource_config) + + def download_to_directory( + self, + cname, + artifacts_dir, + ): + """ + Download S3 artifacts to a given directory. + + :param cname: Canonical name of the GardenLinux S3 artifacts + :param artifacts_dir: Path for the image artifacts + + :since: 0.8.0 + """ + + if not isinstance(artifacts_dir, PathLike): + artifacts_dir = Path(artifacts_dir) + + if not artifacts_dir.is_dir(): + raise RuntimeError(f"Artifacts directory given is invalid: {artifacts_dir}") + + release_object = list( + self._bucket.objects.filter(Prefix=f"meta/singles/{cname}") + )[0] + + self._bucket.download_file( + release_object.key, artifacts_dir.joinpath(f"{cname}.s3_metadata.yaml") + ) + + for s3_object in self._bucket.objects.filter(Prefix=f"objects/{cname}").all(): + self._bucket.download_file( + s3_object.key, artifacts_dir.joinpath(basename(s3_object.key)) + ) + + def upload_from_directory( + self, + cname, + artifacts_dir, + delete_before_push=False, + ): + """ + Pushes S3 artifacts to the underlying bucket. + + :param cname: Canonical name of the GardenLinux S3 artifacts + :param artifacts_dir: Path of the image artifacts + :param delete_before_push: True to delete objects before upload + + :since: 0.8.0 + """ + + if not isinstance(artifacts_dir, PathLike): + artifacts_dir = Path(artifacts_dir) + + cname_object = CName(cname) + + if cname_object.arch is None: + raise RuntimeError("Architecture could not be determined from cname") + + if not artifacts_dir.is_dir(): + raise RuntimeError(f"Artifacts directory given is invalid: {artifacts_dir}") + + release_file = artifacts_dir.joinpath(f"{cname}.release") + release_timestamp = stat(release_file).st_ctime + + release_config = ConfigParser(allow_unnamed_section=True) + release_config.read(release_file) + + if cname_object.version != release_config.get( + UNNAMED_SECTION, "GARDENLINUX_VERSION" + ): + raise RuntimeError( + f"Release file data and given cname conflict detected: Version {cname_object.version}" + ) + + if cname_object.commit_id != release_config.get( + UNNAMED_SECTION, "GARDENLINUX_COMMIT_ID" + ): + raise RuntimeError( + f"Release file data and given cname conflict detected: Commit ID {cname_object.commit_id}" + ) + + commit_hash = release_config.get(UNNAMED_SECTION, "GARDENLINUX_COMMIT_ID_LONG") + + feature_set = release_config.get(UNNAMED_SECTION, "GARDENLINUX_FEATURES") + feature_list = feature_set.split(",") + + metadata = { + "platform": cname_object.platform, + "architecture": cname_object.arch, + "base_image": None, + "build_committish": commit_hash, + "build_timestamp": datetime.fromtimestamp(release_timestamp).isoformat(), + "gardenlinux_epoch": cname_object.version.split(".", 1)[0], + "logs": None, + "modifiers": feature_set, + "require_uefi": "_usi" in feature_list, + "secureboot": "_trustedboot" in feature_list, + "published_image_metadata": None, + "s3_bucket": self._bucket.name, + "s3_key": f"meta/singles/{cname}", + "test_result": None, + "version": cname_object.version, + "paths": [], + } + + for artifact in artifacts_dir.iterdir(): + if not artifact.match(f"{cname}*"): + continue + + s3_key = f"objects/{cname}/{artifact.name}" + + with artifact.open("rb") as fp: + md5sum = file_digest(fp, "md5").hexdigest() + sha256sum = file_digest(fp, "sha256").hexdigest() + + artifact_metadata = { + "name": artifact.name, + "s3_bucket_name": self._bucket.name, + "s3_key": s3_key, + "suffix": "".join(artifact.suffixes), + "md5sum": md5sum, + "sha256sum": sha256sum, + } + + s3_tags = { + "architecture": cname_object.arch, + "platform": cname_object.platform, + "version": cname_object.version, + "committish": commit_hash, + "md5sum": md5sum, + "sha256sum": sha256sum, + } + + if delete_before_push: + self._bucket.delete_objects(Delete={"Objects": [{"Key": s3_key}]}) + + self._bucket.upload_file( + artifact, + s3_key, + ExtraArgs={"Tagging": urlencode(s3_tags)}, + ) + + metadata["paths"].append(artifact_metadata) + + if delete_before_push: + self._bucket.delete_objects( + Delete={"Objects": [{"Key": f"meta/singles/{cname}"}]} + ) + + with TemporaryFile(mode="wb+") as fp: + fp.write(yaml.dump(metadata).encode("utf-8")) + fp.seek(0) + + self._bucket.upload_fileobj( + fp, f"meta/singles/{cname}", ExtraArgs={"ContentType": "text/yaml"} + )