From 22e2f6cd3fec83a83216234e77a99f91b2d3347e Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 9 Jun 2025 12:46:47 +0200 Subject: [PATCH 1/6] add optional registry to config; not used yet --- src/uenv/parse.cpp | 12 ++++++++++++ src/uenv/parse.h | 3 +++ src/uenv/registry.h | 1 + src/uenv/settings.cpp | 25 +++++++++++++++++++++++-- src/uenv/settings.h | 2 ++ 5 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 src/uenv/registry.h diff --git a/src/uenv/parse.cpp b/src/uenv/parse.cpp index 24e5bf8c..a7fd6111 100644 --- a/src/uenv/parse.cpp +++ b/src/uenv/parse.cpp @@ -607,4 +607,16 @@ parse_config_line(const std::string& arg) { return result; } +util::expected +parse_registry_url(const std::string& arg) { + // TODO: parse + // leading [https://] + // name [/name]+ + // where name = [alpha,integer,_,-,.,~]+, see: + // https://blog.sucuri.net/2023/01/bad-paths-the-importance-of-using-valid-url-characters.html + // + // for initial testing, assume that the url is correct + return arg; +} + } // namespace uenv diff --git a/src/uenv/parse.h b/src/uenv/parse.h index 2b7a4a62..7f2c9f4e 100644 --- a/src/uenv/parse.h +++ b/src/uenv/parse.h @@ -67,4 +67,7 @@ parse_registry_entry(const std::string& in); util::expected parse_config_line(const std::string& arg); +util::expected +parse_registry_url(const std::string& arg); + } // namespace uenv diff --git a/src/uenv/registry.h b/src/uenv/registry.h new file mode 100644 index 00000000..6f70f09b --- /dev/null +++ b/src/uenv/registry.h @@ -0,0 +1 @@ +#pragma once diff --git a/src/uenv/settings.cpp b/src/uenv/settings.cpp index ec209c0e..437a6108 100644 --- a/src/uenv/settings.cpp +++ b/src/uenv/settings.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,8 @@ const std::string config_file_default = # set the path to the local uenv repository #repo = /users/bobsmith/uenv +#registry = ... + # by default uenv will choose whether to use color based on your environment. #color=true #color=false @@ -35,6 +38,9 @@ config_base merge(const config_base& lhs, const config_base& rhs) { .repo = lhs.repo ? lhs.repo : rhs.repo ? rhs.repo : std::nullopt, + .registry = lhs.registry ? lhs.registry + : rhs.registry ? rhs.registry + : std::nullopt, .color = lhs.color ? lhs.color : rhs.color ? rhs.color : std::nullopt, @@ -44,6 +50,7 @@ config_base merge(const config_base& lhs, const config_base& rhs) { config_base default_config(const envvars::state& env) { return { .repo = default_repo_path(env), + .registry = site::registry_url(), .color = color::default_color(env), }; } @@ -53,7 +60,8 @@ generate_configuration(const config_base& base) { configuration config; // set the repo path - // initialise to unset + + // initialise to unset, and then set if the path provided by base is valid. config.repo = {}; if (base.repo) { if (auto path = parse_path(base.repo.value())) { @@ -68,6 +76,8 @@ generate_configuration(const config_base& base) { } } + config.registry = base.registry; + // toggle color output config.color = base.color.value_or(false); @@ -186,9 +196,20 @@ read_config_file(const std::filesystem::path& path, } else { return util::unexpected( fmt::format("invalid configuration value '{}={}': color " - "muste be true or false", + "must be true or false", key, value)); } + } else if (key == "registry") { + // an empty value is interpretted as unsetting the registry + if (value == "") { + config.registry = std::nullopt; + } else if (const auto p = parse_registry_url(value)) { + config.registry = p.value(); + } else { + return util::unexpected( + fmt::format("invalid reguistry url '{}={}': {}", key, value, + p.error().message())); + } } else { return util::unexpected( fmt::format("invalid configuration parameter '{}'", key)); diff --git a/src/uenv/settings.h b/src/uenv/settings.h index 3cefb9dc..a568a93b 100644 --- a/src/uenv/settings.h +++ b/src/uenv/settings.h @@ -11,6 +11,7 @@ namespace uenv { struct config_base { std::optional repo; + std::optional registry; std::optional color; }; @@ -36,6 +37,7 @@ config_base merge(const config_base& lhs, const config_base& rhs); struct configuration { std::optional repo; + std::optional registry; bool color; configuration& operator=(const configuration&) = default; }; From 229d84f0e14ec317fd13eae4da5f7fe076e51677 Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 9 Jun 2025 13:01:47 +0200 Subject: [PATCH 2/6] use registry from settings instead of directly using the site value --- .gitignore | 4 ++++ src/cli/copy.cpp | 6 +++++- src/cli/delete.cpp | 6 +++++- src/cli/pull.cpp | 6 +++++- src/cli/push.cpp | 6 +++++- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index ed2558a1..4cdda01e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ install build pyenv + +# npm artifacts installed by tools +package*.json +node_modules diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index 352db24f..f8d72dde 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -164,7 +164,11 @@ int image_copy([[maybe_unused]] const image_copy_args& args, term::error("the destination already exists and will be overwritten"); } - const auto rego_url = site::registry_url(); + if (!settings.config.registry) { + term::error("registry is not configured - set it in the config file or provide --registry option"); + return 1; + } + const auto rego_url = settings.config.registry.value(); spdlog::debug("registry url: {}", rego_url); if (auto result = oras::copy(rego_url, src_label.nspace.value(), src_record, diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index 40e7fa16..e7f0e01a 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -107,7 +107,11 @@ int image_delete([[maybe_unused]] const image_delete_args& args, return 1; } - const auto rego_url = site::registry_url(); + if (!settings.config.registry) { + term::error("registry is not configured - set it in the config file or provide --registry option"); + return 1; + } + const auto rego_url = settings.config.registry.value(); spdlog::debug("registry url: {}", rego_url); for (auto& record : *matches) { auto url = fmt::format( diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index d1c75b5a..380292ab 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -176,7 +176,11 @@ int image_pull([[maybe_unused]] const image_pull_args& args, spdlog::debug("pull meta: {}", pull_meta); spdlog::debug("pull sqfs: {}", pull_sqfs); - auto rego_url = site::registry_url(); + if (!settings.config.registry) { + term::error("registry is not configured - set it in the config file or provide --registry option"); + return 1; + } + auto rego_url = settings.config.registry.value(); spdlog::debug("registry url: {}", rego_url); // the digests returned by oras::discover is a list of artifacts diff --git a/src/cli/push.cpp b/src/cli/push.cpp index 0b4d6604..d68a03fd 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -118,7 +118,11 @@ int image_push([[maybe_unused]] const image_push_args& args, spdlog::info("image_push: squashfs {}", sqfs.value()); try { - auto rego_url = site::registry_url(); + if (!settings.config.registry) { + term::error("registry is not configured - set it in the config file or provide --registry option"); + return 1; + } + auto rego_url = settings.config.registry.value(); spdlog::debug("registry url: {}", rego_url); // Push the SquashFS image From b7c114ffc394a70d938d692695e009399f8de6a6 Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 9 Jun 2025 18:54:11 +0200 Subject: [PATCH 3/6] claudes first pass at generic registry --- meson.build | 1 + src/cli/copy.cpp | 21 +++--- src/cli/delete.cpp | 20 +++++- src/cli/pull.cpp | 111 ++++++++++++++++++++++---------- src/cli/push.cpp | 57 +++++++++++------ src/cli/util.cpp | 25 +++++++- src/cli/util.h | 7 ++ src/site/site.cpp | 25 ++++++++ src/site/site.h | 5 ++ src/uenv/registry.cpp | 145 ++++++++++++++++++++++++++++++++++++++++++ src/uenv/registry.h | 48 ++++++++++++++ src/uenv/settings.cpp | 13 ++++ src/uenv/settings.h | 3 + 13 files changed, 411 insertions(+), 70 deletions(-) create mode 100644 src/uenv/registry.cpp diff --git a/meson.build b/meson.build index 4f4cd65d..a09c0357 100644 --- a/meson.build +++ b/meson.build @@ -46,6 +46,7 @@ lib_src = [ 'src/uenv/oras.cpp', 'src/uenv/parse.cpp', 'src/uenv/print.cpp', + 'src/uenv/registry.cpp', 'src/uenv/repository.cpp', 'src/uenv/settings.cpp', 'src/uenv/uenv.cpp', diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index f8d72dde..a9ec23f8 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,7 @@ #include "copy.h" #include "help.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -91,10 +93,15 @@ int image_copy([[maybe_unused]] const image_copy_args& args, return 1; } - auto src_registry = site::registry_listing(*src_label.nspace); + auto registry_backend = create_registry_from_config(settings.config); + if (!registry_backend) { + term::error("{}", registry_backend.error()); + return 1; + } + + auto src_registry = (*registry_backend)->get_listing(*src_label.nspace); if (!src_registry) { - term::error("unable to get a listing of the uenv", - src_registry.error()); + term::error("unable to get a listing of the uenv: {}", src_registry.error()); return 1; } @@ -154,7 +161,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, spdlog::info("destination record: {} {}", dst_record.sha, dst_record); // check whether the destination already exists - auto dst_registry = site::registry_listing(*dst_label.nspace); + auto dst_registry = (*registry_backend)->get_listing(*dst_label.nspace); if (dst_registry && dst_registry->contains(dst_record)) { if (!args.force) { term::error("the destination already exists - use the --force flag " @@ -164,11 +171,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, term::error("the destination already exists and will be overwritten"); } - if (!settings.config.registry) { - term::error("registry is not configured - set it in the config file or provide --registry option"); - return 1; - } - const auto rego_url = settings.config.registry.value(); + const auto rego_url = (*registry_backend)->get_url(); spdlog::debug("registry url: {}", rego_url); if (auto result = oras::copy(rego_url, src_label.nspace.value(), src_record, diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index e7f0e01a..63c6339e 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include "delete.h" #include "help.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -78,9 +80,20 @@ int image_delete([[maybe_unused]] const image_delete_args& args, } spdlog::debug("requested to delete {}::{}", nspace, label); - auto registry = site::registry_listing(nspace); + auto registry_backend = create_registry_from_config(settings.config); + if (!registry_backend) { + term::error("{}", registry_backend.error()); + return 1; + } + + if (!(*registry_backend)->supports_search()) { + term::error("Registry does not support search - cannot validate uenv for deletion"); + return 1; + } + + auto registry = (*registry_backend)->get_listing(nspace); if (!registry) { - term::error("unable to get a listing of the uenv", registry.error()); + term::error("unable to get a listing of the uenv: {}", registry.error()); return 1; } @@ -95,7 +108,7 @@ int image_delete([[maybe_unused]] const image_delete_args& args, using enum help::block::admonition; term::error("no uenv found that matches '{}'\n\n{}", args.uenv_description, - help::block(info, "try searching for the uenv to copy " + help::block(info, "try searching for the uenv to delete " "first using 'uenv image find'")); return 1; } else if (!matches->unique_sha()) { @@ -113,6 +126,7 @@ int image_delete([[maybe_unused]] const image_delete_args& args, } const auto rego_url = settings.config.registry.value(); spdlog::debug("registry url: {}", rego_url); + for (auto& record : *matches) { auto url = fmt::format( "https://jfrog.svc.cscs.ch/artifactory/uenv/{}/{}/{}/{}/{}/{}", diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index 380292ab..7237c84e 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include "help.h" #include "pull.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -97,37 +99,57 @@ int image_pull([[maybe_unused]] const image_pull_args& args, spdlog::info("image_pull: {}::{}", nspace, label); - auto registry = site::registry_listing(nspace); - if (!registry) { - term::error("unable to get a listing of the uenv", registry.error()); + auto registry_backend = create_registry_from_config(settings.config); + if (!registry_backend) { + term::error("{}", registry_backend.error()); return 1; } + + uenv_record record{}; + + if ((*registry_backend)->supports_search()) { + auto registry = (*registry_backend)->get_listing(nspace); + if (!registry) { + term::error("unable to get a listing of the uenv: {}", registry.error()); + return 1; + } - // search db for matching records - const auto remote_matches = registry->query(label); - if (!remote_matches) { - term::error("invalid search term: {}", registry.error()); - return 1; - } - // check that there is one record with a unique sha - if (remote_matches->empty()) { - using enum help::block::admonition; - term::error("no uenv found that matches '{}'\n\n{}", - args.uenv_description, - help::block(info, "try searching for the uenv to pull " - "first using 'uenv image find'")); - return 1; - } else if (!remote_matches->unique_sha()) { - std::string errmsg = - fmt::format("more than one uenv found that matches '{}':\n", - args.uenv_description); - errmsg += format_record_set(*remote_matches); - term::error("{}", errmsg); - return 1; - } + // search db for matching records + const auto remote_matches = registry->query(label); + if (!remote_matches) { + term::error("invalid search term: {}", registry.error()); + return 1; + } + // check that there is one record with a unique sha + if (remote_matches->empty()) { + using enum help::block::admonition; + term::error("no uenv found that matches '{}'\n\n{}", + args.uenv_description, + help::block(info, "try searching for the uenv to pull " + "first using 'uenv image find'")); + return 1; + } else if (!remote_matches->unique_sha()) { + std::string errmsg = + fmt::format("more than one uenv found that matches '{}':\n", + args.uenv_description); + errmsg += format_record_set(*remote_matches); + term::error("{}", errmsg); + return 1; + } - // pick a record to use for pulling - const auto record = *(remote_matches->begin()); + // pick a record to use for pulling + record = *(remote_matches->begin()); + } else { + spdlog::info("Registry does not support search, proceeding without pre-validation"); + // Construct a minimal record from the label for non-searchable registries + record.name = label.name.value(); + record.version = label.version.value_or("latest"); + record.tag = label.tag.value_or("latest"); + auto system_name = site::get_system_name({}, settings.calling_environment); + record.system = label.system.value_or(system_name.value_or("unknown")); + record.uarch = label.uarch.value_or("unknown"); + // Note: sha will be empty, but ORAS operations should still work + } spdlog::info("pulling {} {}", record.sha, record); // require that a valid repo has been provided @@ -236,15 +258,34 @@ int image_pull([[maybe_unused]] const image_pull_args& args, // add the label to the repo, even if there was no download. // download may have been skipped if a squashfs with the same sha has // been downloaded, and this download uses a different label. - for (auto& r : *remote_matches) { - bool exists = in_repo({.name = r.name, - .version = r.version, - .tag = r.tag, - .system = r.system, - .uarch = r.uarch}); + if ((*registry_backend)->supports_search()) { + auto registry = (*registry_backend)->get_listing(nspace); + if (registry) { + const auto remote_matches = registry->query(label); + if (remote_matches) { + for (auto& r : *remote_matches) { + bool exists = in_repo({.name = r.name, + .version = r.version, + .tag = r.tag, + .system = r.system, + .uarch = r.uarch}); + if (!exists) { + term::msg("updating {}", r); + store->add(r); + } + } + } + } + } else { + // For non-searchable registries, add the constructed record + bool exists = in_repo({.name = record.name, + .version = record.version, + .tag = record.tag, + .system = record.system, + .uarch = record.uarch}); if (!exists) { - term::msg("updating {}", r); - store->add(r); + term::msg("updating {}", record); + store->add(record); } } diff --git a/src/cli/push.cpp b/src/cli/push.cpp index d68a03fd..ff263b35 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -83,30 +84,44 @@ int image_push([[maybe_unused]] const image_push_args& args, dst_label.label); const auto nspace = dst_label.nspace.value(); - auto registry = site::registry_listing(nspace); - if (!registry) { - term::error("unable to get a listing of the uenv", registry.error()); + + auto registry_backend = create_registry_from_config(settings.config); + if (!registry_backend) { + term::error("{}", registry_backend.error()); return 1; } + + if ((*registry_backend)->supports_search()) { + auto registry = (*registry_backend)->get_listing(nspace); + if (!registry) { + term::error("unable to get a listing of the uenv: {}", registry.error()); + return 1; + } - // check whether an image that matches the target label is already - // in the registry. - const auto remote_matches = registry->query(dst_label.label); - if (!remote_matches) { - term::error("invalid search term: {}", registry.error()); - return 1; - } - if (!remote_matches->empty() && !args.force) { - using enum help::block::admonition; - term::error( - "a uenv that matches '{}' is already in the registry\n\n{}", - args.dest, - help::block(info, - "use the --force flag if you want to overwrite it ")); - return 1; - } else if (!remote_matches->empty() && args.force) { - spdlog::info("{} already exists and will be overwritten", args.dest); - term::msg("the destination already exists and will be overwritten"); + // check whether an image that matches the target label is already + // in the registry. + const auto remote_matches = registry->query(dst_label.label); + if (!remote_matches) { + term::error("invalid search term: {}", registry.error()); + return 1; + } + if (!remote_matches->empty() && !args.force) { + using enum help::block::admonition; + term::error( + "a uenv that matches '{}' is already in the registry\n\n{}", + args.dest, + help::block(info, + "use the --force flag if you want to overwrite it ")); + return 1; + } else if (!remote_matches->empty() && args.force) { + spdlog::info("{} already exists and will be overwritten", args.dest); + term::msg("the destination already exists and will be overwritten"); + } + } else { + spdlog::info("Registry does not support search, proceeding without pre-validation"); + if (!args.force) { + term::warn("Registry does not support search - cannot check for existing images. Consider using --force if overwriting is acceptable."); + } } // validate the source squashfs file diff --git a/src/cli/util.cpp b/src/cli/util.cpp index ca18d680..dde0d66c 100644 --- a/src/cli/util.cpp +++ b/src/cli/util.cpp @@ -4,12 +4,13 @@ #include #include +#include +#include +#include #include #include #include -#include - #include "util.h" namespace uenv { @@ -56,4 +57,24 @@ validate_squashfs_image(const std::string& path) { return img; } +util::expected, std::string> +create_registry_from_config(const configuration& config) { + if (!config.registry) { + return util::unexpected("registry is not configured - set it in the config file or provide --registry option"); + } + + auto registry_url = config.registry.value().string(); + + switch (config.registry_type_val) { + case registry_type::site: + return site::create_site_registry(); + case registry_type::oci: + case registry_type::zot: + case registry_type::ghcr: + return create_registry(registry_url, config.registry_type_val); + } + + return util::unexpected("unknown registry type"); +} + } // namespace uenv diff --git a/src/cli/util.h b/src/cli/util.h index 89f32f2c..5f8e5838 100644 --- a/src/cli/util.h +++ b/src/cli/util.h @@ -1,10 +1,13 @@ #pragma once #include +#include #include #include +#include +#include #include namespace uenv { @@ -23,6 +26,10 @@ struct squashfs_image { util::expected validate_squashfs_image(const std::string& path); +// Create registry backend based on configuration +util::expected, std::string> +create_registry_from_config(const configuration& config); + } // namespace uenv template <> class fmt::formatter { diff --git a/src/site/site.cpp b/src/site/site.cpp index 728b400b..835a0e89 100644 --- a/src/site/site.cpp +++ b/src/site/site.cpp @@ -163,4 +163,29 @@ get_credentials(std::optional username, return oras::credentials{.username = uname.value(), .token = token_string}; } +// JFrog-specific registry implementation +class jfrog_registry : public uenv::registry_backend { +public: + util::expected + get_listing(const std::string& nspace) override { + return registry_listing(nspace); + } + + std::string get_url() const override { + return registry_url(); + } + + bool supports_search() const override { + return true; + } + + uenv::registry_type get_type() const override { + return uenv::registry_type::site; + } +}; + +std::unique_ptr create_site_registry() { + return std::make_unique(); +} + } // namespace site diff --git a/src/site/site.h b/src/site/site.h index 0778d022..76a5b03b 100644 --- a/src/site/site.h +++ b/src/site/site.h @@ -1,9 +1,11 @@ #pragma once +#include #include #include #include +#include #include #include #include @@ -29,4 +31,7 @@ util::expected, std::string> get_credentials(std::optional username, std::optional token); +// Create site-specific registry backend (JFrog implementation) +std::unique_ptr create_site_registry(); + } // namespace site diff --git a/src/uenv/registry.cpp b/src/uenv/registry.cpp new file mode 100644 index 00000000..6dde743c --- /dev/null +++ b/src/uenv/registry.cpp @@ -0,0 +1,145 @@ +#include "registry.h" + +#include + +#include + +namespace uenv { + +// Generic OCI Registry implementation +class oci_registry : public registry_backend { +private: + std::string url_; + +public: + explicit oci_registry(const std::string& url) : url_(url) {} + + util::expected + get_listing(const std::string& nspace) override { + spdlog::debug("OCI registry does not support listing for namespace: {}", nspace); + // Return empty repository - OCI registries generally don't support search + return uenv::create_repository(); + } + + std::string get_url() const override { + return url_; + } + + bool supports_search() const override { + return false; + } + + registry_type get_type() const override { + return registry_type::oci; + } +}; + +// Zot Registry implementation +class zot_registry : public registry_backend { +private: + std::string url_; + +public: + explicit zot_registry(const std::string& url) : url_(url) {} + + util::expected + get_listing(const std::string& nspace) override { + spdlog::debug("Zot registry listing for namespace: {}", nspace); + // Zot supports some search capabilities, but for now return empty + // This could be extended to use Zot's GraphQL API + return uenv::create_repository(); + } + + std::string get_url() const override { + return url_; + } + + bool supports_search() const override { + return false; // Could be true if we implement Zot's GraphQL API + } + + registry_type get_type() const override { + return registry_type::zot; + } +}; + +// GitHub Container Registry implementation +class ghcr_registry : public registry_backend { +private: + std::string url_; + +public: + explicit ghcr_registry(const std::string& url) : url_(url) {} + + util::expected + get_listing(const std::string& nspace) override { + spdlog::debug("GHCR registry does not support listing for namespace: {}", nspace); + // GHCR doesn't support comprehensive search via OCI APIs + return uenv::create_repository(); + } + + std::string get_url() const override { + return url_; + } + + bool supports_search() const override { + return false; + } + + registry_type get_type() const override { + return registry_type::ghcr; + } +}; + +// Factory function implementation +std::unique_ptr +create_registry(const std::string& url, registry_type type) { + switch (type) { + case registry_type::oci: + return std::make_unique(url); + case registry_type::zot: + return std::make_unique(url); + case registry_type::ghcr: + return std::make_unique(url); + case registry_type::site: + // Site-specific implementation will be created in site module + spdlog::error("Site registry type should be created via site module"); + return nullptr; + } + return nullptr; +} + +// Parse registry type from string +util::expected +parse_registry_type(const std::string& type_str) { + if (type_str == "oci") { + return registry_type::oci; + } else if (type_str == "zot") { + return registry_type::zot; + } else if (type_str == "ghcr") { + return registry_type::ghcr; + } else if (type_str == "site") { + return registry_type::site; + } else { + return util::unexpected( + "Invalid registry type: " + type_str + + ". Valid types are: oci, zot, ghcr, site"); + } +} + +// Convert registry type to string +std::string registry_type_to_string(registry_type type) { + switch (type) { + case registry_type::oci: + return "oci"; + case registry_type::zot: + return "zot"; + case registry_type::ghcr: + return "ghcr"; + case registry_type::site: + return "site"; + } + return "unknown"; +} + +} // namespace uenv \ No newline at end of file diff --git a/src/uenv/registry.h b/src/uenv/registry.h index 6f70f09b..a21ab770 100644 --- a/src/uenv/registry.h +++ b/src/uenv/registry.h @@ -1 +1,49 @@ #pragma once + +#include +#include +#include + +#include +#include + +namespace uenv { + +enum class registry_type { + oci, // Generic OCI registry (Docker Hub, etc.) + zot, // Zot registry implementation + ghcr, // GitHub Container Registry + site // Site-specific implementation (JFrog, etc.) +}; + +class registry_backend { +public: + virtual ~registry_backend() = default; + + // Get listing of images in a namespace + // Returns empty repository for registries that don't support search + virtual util::expected + get_listing(const std::string& nspace) = 0; + + // Get the registry URL + virtual std::string get_url() const = 0; + + // Whether this registry supports search/listing operations + virtual bool supports_search() const = 0; + + // Get the registry type + virtual registry_type get_type() const = 0; +}; + +// Factory function to create registry backends +std::unique_ptr +create_registry(const std::string& url, registry_type type); + +// Parse registry type from string +util::expected +parse_registry_type(const std::string& type_str); + +// Convert registry type to string +std::string registry_type_to_string(registry_type type); + +} // namespace uenv \ No newline at end of file diff --git a/src/uenv/settings.cpp b/src/uenv/settings.cpp index 437a6108..ecb1b438 100644 --- a/src/uenv/settings.cpp +++ b/src/uenv/settings.cpp @@ -41,6 +41,9 @@ config_base merge(const config_base& lhs, const config_base& rhs) { .registry = lhs.registry ? lhs.registry : rhs.registry ? rhs.registry : std::nullopt, + .registry_type = lhs.registry_type ? lhs.registry_type + : rhs.registry_type ? rhs.registry_type + : std::nullopt, .color = lhs.color ? lhs.color : rhs.color ? rhs.color : std::nullopt, @@ -51,6 +54,7 @@ config_base default_config(const envvars::state& env) { return { .repo = default_repo_path(env), .registry = site::registry_url(), + .registry_type = "site", .color = color::default_color(env), }; } @@ -78,6 +82,13 @@ generate_configuration(const config_base& base) { config.registry = base.registry; + // set registry type + auto registry_type_result = parse_registry_type(base.registry_type.value_or("site")); + if (!registry_type_result) { + return util::unexpected(registry_type_result.error()); + } + config.registry_type_val = *registry_type_result; + // toggle color output config.color = base.color.value_or(false); @@ -210,6 +221,8 @@ read_config_file(const std::filesystem::path& path, fmt::format("invalid reguistry url '{}={}': {}", key, value, p.error().message())); } + } else if (key == "registry_type") { + config.registry_type = value; } else { return util::unexpected( fmt::format("invalid configuration parameter '{}'", key)); diff --git a/src/uenv/settings.h b/src/uenv/settings.h index a568a93b..b6fe2ff5 100644 --- a/src/uenv/settings.h +++ b/src/uenv/settings.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -12,6 +13,7 @@ namespace uenv { struct config_base { std::optional repo; std::optional registry; + std::optional registry_type; std::optional color; }; @@ -38,6 +40,7 @@ config_base merge(const config_base& lhs, const config_base& rhs); struct configuration { std::optional repo; std::optional registry; + registry_type registry_type_val; bool color; configuration& operator=(const configuration&) = default; }; From 2e0628bc1e6a86a1046762d7c64321ad61ef4bbf Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 10 Jun 2025 07:32:49 +0200 Subject: [PATCH 4/6] wip on integration tests with registries --- src/cli/copy.cpp | 11 ++-- src/cli/delete.cpp | 19 +++--- src/cli/find.cpp | 17 ++++- src/cli/pull.cpp | 27 ++++---- src/cli/push.cpp | 28 ++++---- src/cli/util.cpp | 23 +++---- src/cli/util.h | 4 +- src/site/site.cpp | 23 ++++--- src/site/site.h | 9 +-- src/uenv/registry.cpp | 122 +++++++++++++++++++---------------- src/uenv/registry.h | 117 +++++++++++++++++++++++++++------ src/uenv/settings.cpp | 5 +- test/integration/cli.bats | 74 +++++++++++++++++++++ test/integration/common.bash | 54 ++++++++++++++++ 14 files changed, 384 insertions(+), 149 deletions(-) diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index a9ec23f8..b72981bd 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -98,10 +98,11 @@ int image_copy([[maybe_unused]] const image_copy_args& args, term::error("{}", registry_backend.error()); return 1; } - - auto src_registry = (*registry_backend)->get_listing(*src_label.nspace); + + auto src_registry = registry_backend->get_listing(*src_label.nspace); if (!src_registry) { - term::error("unable to get a listing of the uenv: {}", src_registry.error()); + term::error("unable to get a listing of the uenv: {}", + src_registry.error()); return 1; } @@ -161,7 +162,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, spdlog::info("destination record: {} {}", dst_record.sha, dst_record); // check whether the destination already exists - auto dst_registry = (*registry_backend)->get_listing(*dst_label.nspace); + auto dst_registry = registry_backend->get_listing(*dst_label.nspace); if (dst_registry && dst_registry->contains(dst_record)) { if (!args.force) { term::error("the destination already exists - use the --force flag " @@ -171,7 +172,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, term::error("the destination already exists and will be overwritten"); } - const auto rego_url = (*registry_backend)->get_url(); + const auto rego_url = registry_backend->get_url(); spdlog::debug("registry url: {}", rego_url); if (auto result = oras::copy(rego_url, src_label.nspace.value(), src_record, diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index 63c6339e..c745f095 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -85,15 +85,17 @@ int image_delete([[maybe_unused]] const image_delete_args& args, term::error("{}", registry_backend.error()); return 1; } - - if (!(*registry_backend)->supports_search()) { - term::error("Registry does not support search - cannot validate uenv for deletion"); + + if (!registry_backend->supports_search()) { + term::error("Registry does not support search - cannot validate uenv " + "for deletion"); return 1; } - - auto registry = (*registry_backend)->get_listing(nspace); + + auto registry = registry_backend->get_listing(nspace); if (!registry) { - term::error("unable to get a listing of the uenv: {}", registry.error()); + term::error("unable to get a listing of the uenv: {}", + registry.error()); return 1; } @@ -121,12 +123,13 @@ int image_delete([[maybe_unused]] const image_delete_args& args, } if (!settings.config.registry) { - term::error("registry is not configured - set it in the config file or provide --registry option"); + term::error("registry is not configured - set it in the config file or " + "provide --registry option"); return 1; } const auto rego_url = settings.config.registry.value(); spdlog::debug("registry url: {}", rego_url); - + for (auto& record : *matches) { auto url = fmt::format( "https://jfrog.svc.cscs.ch/artifactory/uenv/{}/{}/{}/{}/{}/{}", diff --git a/src/cli/find.cpp b/src/cli/find.cpp index 87d9b66a..4db5f28a 100644 --- a/src/cli/find.cpp +++ b/src/cli/find.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include "find.h" #include "help.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -67,9 +69,20 @@ int image_find([[maybe_unused]] const image_find_args& args, site::get_system_name(label.system, settings.calling_environment); spdlog::info("image_find: {}::{}", nspace, label); - auto store = site::registry_listing(nspace); + auto registry = create_registry_from_config(settings.config); + if (!registry) { + term::error("{}", registry.error()); + return 1; + } + + if (!registry->supports_search()) { + term::error("Registry does not support search functionality"); + return 1; + } + + auto store = registry->get_listing(nspace); if (!store) { - term::error("unable to get a listing of the uenv", store.error()); + term::error("unable to get a listing of the uenv: {}", store.error()); return 1; } diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index 7237c84e..e6b6450d 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -104,13 +104,14 @@ int image_pull([[maybe_unused]] const image_pull_args& args, term::error("{}", registry_backend.error()); return 1; } - + uenv_record record{}; - - if ((*registry_backend)->supports_search()) { - auto registry = (*registry_backend)->get_listing(nspace); + + if (registry_backend->supports_search()) { + auto registry = registry_backend->get_listing(nspace); if (!registry) { - term::error("unable to get a listing of the uenv: {}", registry.error()); + term::error("unable to get a listing of the uenv: {}", + registry.error()); return 1; } @@ -140,12 +141,15 @@ int image_pull([[maybe_unused]] const image_pull_args& args, // pick a record to use for pulling record = *(remote_matches->begin()); } else { - spdlog::info("Registry does not support search, proceeding without pre-validation"); - // Construct a minimal record from the label for non-searchable registries + spdlog::info("Registry does not support search, proceeding without " + "pre-validation"); + // Construct a minimal record from the label for non-searchable + // registries record.name = label.name.value(); record.version = label.version.value_or("latest"); record.tag = label.tag.value_or("latest"); - auto system_name = site::get_system_name({}, settings.calling_environment); + auto system_name = + site::get_system_name({}, settings.calling_environment); record.system = label.system.value_or(system_name.value_or("unknown")); record.uarch = label.uarch.value_or("unknown"); // Note: sha will be empty, but ORAS operations should still work @@ -199,7 +203,8 @@ int image_pull([[maybe_unused]] const image_pull_args& args, spdlog::debug("pull sqfs: {}", pull_sqfs); if (!settings.config.registry) { - term::error("registry is not configured - set it in the config file or provide --registry option"); + term::error("registry is not configured - set it in the config " + "file or provide --registry option"); return 1; } auto rego_url = settings.config.registry.value(); @@ -258,8 +263,8 @@ int image_pull([[maybe_unused]] const image_pull_args& args, // add the label to the repo, even if there was no download. // download may have been skipped if a squashfs with the same sha has // been downloaded, and this download uses a different label. - if ((*registry_backend)->supports_search()) { - auto registry = (*registry_backend)->get_listing(nspace); + if (registry_backend->supports_search()) { + auto registry = registry_backend->get_listing(nspace); if (registry) { const auto remote_matches = registry->query(label); if (remote_matches) { diff --git a/src/cli/push.cpp b/src/cli/push.cpp index ff263b35..66f6a158 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -84,17 +84,18 @@ int image_push([[maybe_unused]] const image_push_args& args, dst_label.label); const auto nspace = dst_label.nspace.value(); - + auto registry_backend = create_registry_from_config(settings.config); if (!registry_backend) { term::error("{}", registry_backend.error()); return 1; } - - if ((*registry_backend)->supports_search()) { - auto registry = (*registry_backend)->get_listing(nspace); + + if (registry_backend->supports_search()) { + auto registry = registry_backend->get_listing(nspace); if (!registry) { - term::error("unable to get a listing of the uenv: {}", registry.error()); + term::error("unable to get a listing of the uenv: {}", + registry.error()); return 1; } @@ -110,17 +111,21 @@ int image_push([[maybe_unused]] const image_push_args& args, term::error( "a uenv that matches '{}' is already in the registry\n\n{}", args.dest, - help::block(info, - "use the --force flag if you want to overwrite it ")); + help::block( + info, "use the --force flag if you want to overwrite it ")); return 1; } else if (!remote_matches->empty() && args.force) { - spdlog::info("{} already exists and will be overwritten", args.dest); + spdlog::info("{} already exists and will be overwritten", + args.dest); term::msg("the destination already exists and will be overwritten"); } } else { - spdlog::info("Registry does not support search, proceeding without pre-validation"); + spdlog::info("Registry does not support search, proceeding without " + "pre-validation"); if (!args.force) { - term::warn("Registry does not support search - cannot check for existing images. Consider using --force if overwriting is acceptable."); + term::warn( + "Registry does not support search - cannot check for existing " + "images. Consider using --force if overwriting is acceptable."); } } @@ -134,7 +139,8 @@ int image_push([[maybe_unused]] const image_push_args& args, try { if (!settings.config.registry) { - term::error("registry is not configured - set it in the config file or provide --registry option"); + term::error("registry is not configured - set it in the config " + "file or provide --registry option"); return 1; } auto rego_url = settings.config.registry.value(); diff --git a/src/cli/util.cpp b/src/cli/util.cpp index dde0d66c..6c9fb653 100644 --- a/src/cli/util.cpp +++ b/src/cli/util.cpp @@ -57,23 +57,24 @@ validate_squashfs_image(const std::string& path) { return img; } -util::expected, std::string> +util::expected create_registry_from_config(const configuration& config) { if (!config.registry) { - return util::unexpected("registry is not configured - set it in the config file or provide --registry option"); + return util::unexpected("registry is not configured - set it in the " + "config file or provide --registry option"); } - + auto registry_url = config.registry.value().string(); - + switch (config.registry_type_val) { - case registry_type::site: - return site::create_site_registry(); - case registry_type::oci: - case registry_type::zot: - case registry_type::ghcr: - return create_registry(registry_url, config.registry_type_val); + case registry_type::site: + return site::create_site_registry(); + case registry_type::oci: + case registry_type::zot: + case registry_type::ghcr: + return create_registry(registry_url, config.registry_type_val); } - + return util::unexpected("unknown registry type"); } diff --git a/src/cli/util.h b/src/cli/util.h index 5f8e5838..7210f267 100644 --- a/src/cli/util.h +++ b/src/cli/util.h @@ -26,8 +26,8 @@ struct squashfs_image { util::expected validate_squashfs_image(const std::string& path); -// Create registry backend based on configuration -util::expected, std::string> +// Create registry based on configuration +util::expected create_registry_from_config(const configuration& config); } // namespace uenv diff --git a/src/site/site.cpp b/src/site/site.cpp index 835a0e89..e033aa4d 100644 --- a/src/site/site.cpp +++ b/src/site/site.cpp @@ -49,7 +49,7 @@ std::string default_namespace() { return "deploy"; } -util::expected +static util::expected registry_listing(const std::string& nspace) { using json = nlohmann::json; @@ -112,7 +112,7 @@ registry_listing(const std::string& nspace) { return store; } -std::string registry_url() { +static std::string registry_url() { return "jfrog.svc.cscs.ch/uenv"; } @@ -163,29 +163,28 @@ get_credentials(std::optional username, return oras::credentials{.username = uname.value(), .token = token_string}; } -// JFrog-specific registry implementation -class jfrog_registry : public uenv::registry_backend { -public: - util::expected - get_listing(const std::string& nspace) override { +// cscs-specific registry implementation +struct cscs_registry { + util::expected + get_listing(const std::string& nspace) const { return registry_listing(nspace); } - std::string get_url() const override { + std::string get_url() const { return registry_url(); } - bool supports_search() const override { + bool supports_search() const { return true; } - uenv::registry_type get_type() const override { + uenv::registry_type get_type() const { return uenv::registry_type::site; } }; -std::unique_ptr create_site_registry() { - return std::make_unique(); +uenv::registry create_site_registry() { + return uenv::registry{cscs_registry{}}; } } // namespace site diff --git a/src/site/site.h b/src/site/site.h index 76a5b03b..6ea9e237 100644 --- a/src/site/site.h +++ b/src/site/site.h @@ -22,16 +22,11 @@ std::optional get_username(); // default namespace for image deployment std::string default_namespace(); -util::expected -registry_listing(const std::string& nspace); - -std::string registry_url(); - util::expected, std::string> get_credentials(std::optional username, std::optional token); -// Create site-specific registry backend (JFrog implementation) -std::unique_ptr create_site_registry(); +// Create site-specific registry (JFrog implementation) +uenv::registry create_site_registry(); } // namespace site diff --git a/src/uenv/registry.cpp b/src/uenv/registry.cpp index 6dde743c..53ad7236 100644 --- a/src/uenv/registry.cpp +++ b/src/uenv/registry.cpp @@ -1,5 +1,6 @@ #include "registry.h" +#include #include #include @@ -7,110 +8,117 @@ namespace uenv { // Generic OCI Registry implementation -class oci_registry : public registry_backend { -private: +struct oci_registry { + private: std::string url_; -public: - explicit oci_registry(const std::string& url) : url_(url) {} + public: + explicit oci_registry(const std::string& url) : url_(url) { + } - util::expected - get_listing(const std::string& nspace) override { - spdlog::debug("OCI registry does not support listing for namespace: {}", nspace); - // Return empty repository - OCI registries generally don't support search + util::expected + get_listing(const std::string& nspace) const { + spdlog::debug("OCI registry does not support listing for namespace: {}", + nspace); + // Return empty repository - OCI registries generally don't support + // search return uenv::create_repository(); } - std::string get_url() const override { + std::string get_url() const { return url_; } - bool supports_search() const override { + bool supports_search() const { return false; } - registry_type get_type() const override { + registry_type get_type() const { return registry_type::oci; } }; // Zot Registry implementation -class zot_registry : public registry_backend { -private: +struct zot_registry { + private: std::string url_; -public: - explicit zot_registry(const std::string& url) : url_(url) {} + public: + explicit zot_registry(const std::string& url) : url_(url) { + } - util::expected - get_listing(const std::string& nspace) override { + util::expected + get_listing(const std::string& nspace) const { spdlog::debug("Zot registry listing for namespace: {}", nspace); // Zot supports some search capabilities, but for now return empty // This could be extended to use Zot's GraphQL API return uenv::create_repository(); } - std::string get_url() const override { + std::string get_url() const { return url_; } - bool supports_search() const override { + bool supports_search() const { return false; // Could be true if we implement Zot's GraphQL API } - registry_type get_type() const override { + registry_type get_type() const { return registry_type::zot; } }; // GitHub Container Registry implementation -class ghcr_registry : public registry_backend { -private: +struct ghcr_registry { + private: std::string url_; -public: - explicit ghcr_registry(const std::string& url) : url_(url) {} + public: + explicit ghcr_registry(const std::string& url) : url_(url) { + } - util::expected - get_listing(const std::string& nspace) override { - spdlog::debug("GHCR registry does not support listing for namespace: {}", nspace); + util::expected + get_listing(const std::string& nspace) const { + spdlog::debug( + "GHCR registry does not support listing for namespace: {}", nspace); // GHCR doesn't support comprehensive search via OCI APIs return uenv::create_repository(); } - std::string get_url() const override { + std::string get_url() const { return url_; } - bool supports_search() const override { + bool supports_search() const { return false; } - registry_type get_type() const override { + registry_type get_type() const { return registry_type::ghcr; } }; // Factory function implementation -std::unique_ptr -create_registry(const std::string& url, registry_type type) { +registry create_registry(const std::string& url, registry_type type) { switch (type) { - case registry_type::oci: - return std::make_unique(url); - case registry_type::zot: - return std::make_unique(url); - case registry_type::ghcr: - return std::make_unique(url); - case registry_type::site: - // Site-specific implementation will be created in site module - spdlog::error("Site registry type should be created via site module"); - return nullptr; - } - return nullptr; + case registry_type::oci: + return registry{oci_registry{url}}; + case registry_type::zot: + return registry{zot_registry{url}}; + case registry_type::ghcr: + return registry{ghcr_registry{url}}; + case registry_type::site: + // Site-specific implementation will be created in site module + spdlog::error("Site registry type should be created via site module"); + // Return a default OCI registry as fallback + return registry{oci_registry{url}}; + } + // Should never reach here, but provide a fallback + return registry{oci_registry{url}}; } // Parse registry type from string -util::expected +util::expected parse_registry_type(const std::string& type_str) { if (type_str == "oci") { return registry_type::oci; @@ -121,25 +129,25 @@ parse_registry_type(const std::string& type_str) { } else if (type_str == "site") { return registry_type::site; } else { - return util::unexpected( - "Invalid registry type: " + type_str + - ". Valid types are: oci, zot, ghcr, site"); + return util::unexpected(fmt::format( + "Invalid registry type: {}. Valid types are: oci, zot, ghcr, site", + type_str)); } } // Convert registry type to string std::string registry_type_to_string(registry_type type) { switch (type) { - case registry_type::oci: - return "oci"; - case registry_type::zot: - return "zot"; - case registry_type::ghcr: - return "ghcr"; - case registry_type::site: - return "site"; + case registry_type::oci: + return "oci"; + case registry_type::zot: + return "zot"; + case registry_type::ghcr: + return "ghcr"; + case registry_type::site: + return "site"; } return "unknown"; } -} // namespace uenv \ No newline at end of file +} // namespace uenv diff --git a/src/uenv/registry.h b/src/uenv/registry.h index a21ab770..a02721bc 100644 --- a/src/uenv/registry.h +++ b/src/uenv/registry.h @@ -10,40 +10,115 @@ namespace uenv { enum class registry_type { - oci, // Generic OCI registry (Docker Hub, etc.) - zot, // Zot registry implementation - ghcr, // GitHub Container Registry - site // Site-specific implementation (JFrog, etc.) + oci, // Generic OCI registry (Docker Hub, etc.) + zot, // Zot registry implementation + ghcr, // GitHub Container Registry + site // Site-specific implementation (JFrog, etc.) }; -class registry_backend { -public: - virtual ~registry_backend() = default; - +// Concept for registry implementations +// Any type T that implements these operations can be used as a registry +template +concept RegistryImpl = requires(T registry, const std::string& nspace) { + { + registry.get_listing(nspace) + } -> std::convertible_to>; + { registry.get_url() } -> std::convertible_to; + { registry.supports_search() } -> std::convertible_to; + { registry.get_type() } -> std::convertible_to; +}; + +// Type-erased registry class using value semantics +class registry { + public: + template + registry(T impl) : impl_(std::make_unique>(std::move(impl))) { + } + + registry(registry&& other) = default; + + registry(const registry& other) : impl_(other.impl_->clone()) { + } + + registry& operator=(registry&& other) = default; + registry& operator=(const registry& other) { + return *this = registry(other); + } + // Get listing of images in a namespace // Returns empty repository for registries that don't support search - virtual util::expected - get_listing(const std::string& nspace) = 0; - + util::expected + get_listing(const std::string& nspace) const { + return impl_->get_listing(nspace); + } + // Get the registry URL - virtual std::string get_url() const = 0; - + std::string get_url() const { + return impl_->get_url(); + } + // Whether this registry supports search/listing operations - virtual bool supports_search() const = 0; - + bool supports_search() const { + return impl_->supports_search(); + } + // Get the registry type - virtual registry_type get_type() const = 0; + registry_type get_type() const { + return impl_->get_type(); + } + + private: + struct interface { + virtual ~interface() = default; + virtual std::unique_ptr clone() = 0; + virtual util::expected + get_listing(const std::string& nspace) const = 0; + virtual std::string get_url() const = 0; + virtual bool supports_search() const = 0; + virtual registry_type get_type() const = 0; + }; + + std::unique_ptr impl_; + + template struct wrap : interface { + explicit wrap(const T& impl) : wrapped(impl) { + } + explicit wrap(T&& impl) : wrapped(std::move(impl)) { + } + + virtual std::unique_ptr clone() override { + return std::make_unique>(wrapped); + } + + virtual util::expected + get_listing(const std::string& nspace) const override { + return wrapped.get_listing(nspace); + } + + virtual std::string get_url() const override { + return wrapped.get_url(); + } + + virtual bool supports_search() const override { + return wrapped.supports_search(); + } + + virtual registry_type get_type() const override { + return wrapped.get_type(); + } + + T wrapped; + }; }; -// Factory function to create registry backends -std::unique_ptr -create_registry(const std::string& url, registry_type type); +// Factory function to create registry implementations +registry create_registry(const std::string& url, registry_type type); // Parse registry type from string -util::expected +util::expected parse_registry_type(const std::string& type_str); // Convert registry type to string std::string registry_type_to_string(registry_type type); -} // namespace uenv \ No newline at end of file +} // namespace uenv diff --git a/src/uenv/settings.cpp b/src/uenv/settings.cpp index ecb1b438..382af5ec 100644 --- a/src/uenv/settings.cpp +++ b/src/uenv/settings.cpp @@ -53,7 +53,7 @@ config_base merge(const config_base& lhs, const config_base& rhs) { config_base default_config(const envvars::state& env) { return { .repo = default_repo_path(env), - .registry = site::registry_url(), + .registry = "jfrog.svc.cscs.ch/uenv", // Default site registry URL .registry_type = "site", .color = color::default_color(env), }; @@ -83,7 +83,8 @@ generate_configuration(const config_base& base) { config.registry = base.registry; // set registry type - auto registry_type_result = parse_registry_type(base.registry_type.value_or("site")); + auto registry_type_result = + parse_registry_type(base.registry_type.value_or("site")); if (!registry_type_result) { return util::unexpected(registry_type_result.error()); } diff --git a/test/integration/cli.bats b/test/integration/cli.bats index 64574bfe..99ab7b0f 100644 --- a/test/integration/cli.bats +++ b/test/integration/cli.bats @@ -467,3 +467,77 @@ EOF [ ! -d $UENV_REPO_PATH/images/$sha ] } +@test "docker registry setup teardown" { + # Skip if Docker is not available + if ! registry_is_available; then + skip "Docker not available for registry tests" + fi + + # Test registry setup + setup_test_registry + + # Verify registry is accessible + run curl -s "http://localhost:$REGISTRY_PORT/v2/" + assert_success + + # Verify environment variables are set + [ -n "$REGISTRY_URL" ] + [ -n "$REGISTRY_PORT" ] + [ -n "$REGISTRY_CONTAINER" ] + + # Store container name before teardown + local container_name="$REGISTRY_CONTAINER" + + # Test teardown + teardown_test_registry + + # Verify container is removed + run docker ps -a --filter "name=$container_name" --format "{{.Names}}" + refute_output --partial "$container_name" +} + +@test "zot registry push pull" { + # Skip if Docker is not available + if ! registry_is_available; then + skip "Docker not available for registry tests" + fi + + # Set up test registry + setup_test_registry + + # Create a test repository for the source image + local src_repo=$(mktemp -d $TMP/push-test-XXXXXX) + run uenv repo create $src_repo + assert_success + + # Add an app image to the source repo + run uenv --repo=$src_repo image add test-app/1.0:v1@test%zen3 $SQFS_LIB/apptool/standalone/app42.squashfs + assert_success + + # Push the image to the Zot registry + run uenv --repo=$src_repo --registry=$REGISTRY_URL --registry-type=zot \ + image push test-app/1.0:v1@test%zen3 test::test-app/1.0:v1@test%zen3 + assert_success + assert_output --partial "successfully pushed" + + # Create a separate repository for pulling + local dst_repo=$(mktemp -d $TMP/pull-test-XXXXXX) + run uenv repo create $dst_repo + assert_success + + # Pull the image from the Zot registry + run uenv --repo=$dst_repo --registry=$REGISTRY_URL --registry-type=zot \ + image pull test::test-app/1.0:v1@test%zen3 + assert_success + + # Verify the image was pulled successfully + run uenv --repo=$dst_repo image ls test-app --no-header + assert_success + assert_output --partial "test-app/1.0:v1" + assert_output --partial "zen3" + assert_output --partial "test" + + # Clean up + teardown_test_registry +} + diff --git a/test/integration/common.bash b/test/integration/common.bash index 9dd466b7..5db1d575 100755 --- a/test/integration/common.bash +++ b/test/integration/common.bash @@ -33,3 +33,57 @@ function run_sbatch() { run_sbatch_unchecked "$@" [ "${status}" -eq 0 ] } + +# Docker-based test registry setup and teardown functions +function setup_test_registry() { + # Check if Docker is available + if ! command -v docker &> /dev/null; then + skip "Docker not available for registry tests" + fi + + # Use a random high port to avoid conflicts + export REGISTRY_PORT=$((5000 + RANDOM % 1000)) + export REGISTRY_URL="localhost:$REGISTRY_PORT" + export REGISTRY_CONTAINER="uenv-test-registry-$$" + + log "Setting up test registry on port $REGISTRY_PORT" + + # Start Zot registry container + docker run -d \ + --name "$REGISTRY_CONTAINER" \ + -p "$REGISTRY_PORT:5000" \ + ghcr.io/project-zot/zot:latest || { + skip "Failed to start Docker registry container" + } + + # Wait for registry to be ready (up to 30 seconds) + local count=0 + while [ $count -lt 30 ]; do + if curl -s "http://localhost:$REGISTRY_PORT/v2/" > /dev/null 2>&1; then + log "Test registry is ready at $REGISTRY_URL" + return 0 + fi + sleep 1 + count=$((count + 1)) + done + + # If we get here, the registry didn't start properly + docker logs "$REGISTRY_CONTAINER" || true + teardown_test_registry + skip "Test registry failed to become ready" +} + +function teardown_test_registry() { + if [ -n "${REGISTRY_CONTAINER:-}" ]; then + log "Cleaning up test registry container: $REGISTRY_CONTAINER" + docker stop "$REGISTRY_CONTAINER" >/dev/null 2>&1 || true + docker rm "$REGISTRY_CONTAINER" >/dev/null 2>&1 || true + unset REGISTRY_CONTAINER + unset REGISTRY_PORT + unset REGISTRY_URL + fi +} + +function registry_is_available() { + command -v docker &> /dev/null +} From 315b3a4668d6eeea7c3dea1f322db251720fa35f Mon Sep 17 00:00:00 2001 From: bcumming Date: Wed, 11 Jun 2025 14:11:32 +0200 Subject: [PATCH 5/6] set up a registry for each bats cli test; always tear down on failed tests --- CLAUDE.md | 110 ++++++++++++++++++++++++++++++++++++++ test/integration/cli.bats | 14 ++--- 2 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..5d45759f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,110 @@ +# Registry Abstraction Implementation Status + +## Context +Refactoring hardcoded site-specific JFrog registry to support multiple registry types (OCI, Zot, GHCR, site-specific). Goal is to allow users to configure different registries while maintaining backward compatibility. + +## Key Understanding +- **Repository**: Local disk storage (SQLite + subdirectories) for downloaded images +- **Registry**: Remote OCI container registry (JFrog, GHCR, Docker Hub, etc.) +- **Repository data structure**: Also used as in-memory container for registry search results (consistent query interface) + +## Completed Work ✅ + +### 1. Core Abstraction (`src/uenv/registry.h` + `src/uenv/registry.cpp`) +- Created `registry_backend` abstract base class +- Implemented `registry_type` enum (oci, zot, ghcr, site) +- Added factory function `create_registry(url, type)` +- OCI/Zot/GHCR implementations return empty repositories (no search support) +- Added `supports_search()` capability flag + +### 2. Site Integration (`src/site/site.h` + `src/site/site.cpp`) +- Added `jfrog_registry` class wrapping existing `registry_listing()` function +- Added `create_site_registry()` factory function +- Preserves existing JFrog custom API behavior + +### 3. Configuration (`src/uenv/settings.h` + `src/uenv/settings.cpp`) +- Added `registry_type` field to `config_base` and `configuration` +- Added parsing for registry_type config parameter +- Default type is "site" for backward compatibility + +### 4. CLI Utilities (`src/cli/util.h` + `src/cli/util.cpp`) +- Added `create_registry_from_config()` helper function +- Handles registry backend creation based on configuration + +### 5. Build System (`meson.build`) +- Added `src/uenv/registry.cpp` to lib_src + +### 6. Complete CLI Updates ✅ +- Updated `src/cli/copy.cpp` to use new registry abstraction +- Updated `src/cli/delete.cpp` to use registry abstraction with graceful degradation +- Updated `src/cli/pull.cpp` to use registry abstraction with graceful degradation +- Updated `src/cli/push.cpp` to use registry abstraction with graceful degradation +- ✅ All CLI files now use the registry abstraction +- ✅ Code compiles and tests pass + +### 7. Type Erasure Refactoring ✅ +- Refactored registry from abstract base class to type-erased value class +- Used concept-based approach similar to `help::item` pattern +- Registry implementations are now structs that satisfy `RegistryImpl` concept +- Registry objects can be stored by value, copied, and moved +- Updated all CLI utilities to use value-based API instead of pointer-based +- ✅ All code compiles and tests pass + +## Remaining Work 🚧 + +### 2. Implement Graceful Degradation Pattern ✅ +All CLI commands now implement graceful degradation: +- **delete.cpp**: Requires search support, fails gracefully if not available +- **pull.cpp**: Constructs minimal record for non-searchable registries +- **push.cpp**: Skips existence check for non-searchable registries with warning +- **copy.cpp**: Uses search for validation when available + +Pattern implemented: +```cpp +auto registry_backend = create_registry_from_config(settings.config); +if (registry_backend->supports_search()) { + // Existing logic: validate via listing first + auto listing = registry_backend->get_listing(nspace); + // ... validation, disambiguation, etc. +} else { + // Graceful degradation: proceed without validation + spdlog::info("Registry does not support search, proceeding without pre-validation"); + // ... direct ORAS operations with proper error handling +} +``` + +### 3. Enhance ORAS Error Handling +Ensure ORAS wrapper functions return well-structured errors: +- "Image not found" errors +- "Network/registry unreachable" errors +- "Authentication failed" errors +- "Invalid image format" errors + +### 4. Update Error Messages +- With search: "No image found matching 'X' in registry listing" +- Without search: "Failed to pull image 'X': image not found in registry" + +### 5. Testing +- Test searchable registry (JFrog): Existing behavior preserved +- Test non-searchable registry: Graceful operation with proper errors + +## Configuration Examples +``` +# Site-specific (current default) +registry = jfrog.svc.cscs.ch/uenv +registry_type = site + +# GitHub Container Registry +registry = ghcr.io +registry_type = ghcr + +# Generic OCI registry +registry = docker.io +registry_type = oci +``` + +## Next Steps +1. Complete CLI file updates with supports_search() branching +2. Enhance ORAS error handling +3. Test with different registry types +4. Update documentation \ No newline at end of file diff --git a/test/integration/cli.bats b/test/integration/cli.bats index 99ab7b0f..a5f43d27 100644 --- a/test/integration/cli.bats +++ b/test/integration/cli.bats @@ -23,10 +23,13 @@ function setup() { # remove the bash function uenv, if an older version of uenv is installed on # the system unset -f uenv + + # Set up test registry + setup_test_registry } function teardown() { - : + teardown_test_registry } @test "noargs" { @@ -473,9 +476,6 @@ EOF skip "Docker not available for registry tests" fi - # Test registry setup - setup_test_registry - # Verify registry is accessible run curl -s "http://localhost:$REGISTRY_PORT/v2/" assert_success @@ -502,9 +502,6 @@ EOF skip "Docker not available for registry tests" fi - # Set up test registry - setup_test_registry - # Create a test repository for the source image local src_repo=$(mktemp -d $TMP/push-test-XXXXXX) run uenv repo create $src_repo @@ -536,8 +533,5 @@ EOF assert_output --partial "test-app/1.0:v1" assert_output --partial "zen3" assert_output --partial "test" - - # Clean up - teardown_test_registry } From 4f2f1978de4fb5e9cb3f5ffb7efcb752a2f3a9c7 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 19 Jun 2025 09:29:47 +0200 Subject: [PATCH 6/6] wip on oras refactor --- oras.md | 184 ++++++++++++++++++++++++++++++++++++++ src/cli/copy.cpp | 6 +- src/cli/delete.cpp | 2 +- src/cli/find.cpp | 6 +- src/cli/pull.cpp | 62 +++++++------ src/cli/push.cpp | 2 +- src/cli/uenv.cpp | 12 ++- src/cli/util.cpp | 2 - src/site/site.cpp | 12 ++- src/uenv/oras.cpp | 131 ++++++++++++++++++++------- src/uenv/oras.h | 11 +++ src/uenv/parse.cpp | 54 +++++++++++ src/uenv/parse.h | 3 + src/uenv/registry.cpp | 101 +++------------------ src/uenv/registry.h | 101 +++++++++++++++------ src/uenv/settings.cpp | 85 ++++++++++-------- src/uenv/settings.h | 24 ++++- test/integration/cli.bats | 47 +++++----- test/unit/parse.cpp | 49 ++++++++-- 19 files changed, 639 insertions(+), 255 deletions(-) create mode 100644 oras.md diff --git a/oras.md b/oras.md new file mode 100644 index 00000000..85da136e --- /dev/null +++ b/oras.md @@ -0,0 +1,184 @@ + +The *manifest* for a uenv squashfs is a JSON file that can be obtained using `oras manifest fetch rego/tag` + +* there is always the same empty `config` field + * `e30=` is an empty JSON object `{}` in base64 encoding +* there is a single layer: the squashfs file +* there is a single annotation with the time of creation + +```console +$ oras manifest fetch localhost:5862/deploy/cluster/zen3/app/1.0:v1 | jq . +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/x-squashfs", + "config": { + "mediaType": "application/vnd.oci.empty.v1+json", + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "size": 2, + "data": "e30=" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "digest": "sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db", + "size": 4096, + "annotations": { + "org.opencontainers.image.title": "app42.squashfs" + } + } + ], + "annotations": { + "org.opencontainers.image.created": "2025-06-17T08:32:08Z" + } +} + +``` + +Every file (squashfs, tar ball of meta path, manifest.json, etc) in a registry has a *digest* of the form `sha256:<64 character sha>`. + +* the config json has a digest that is `printf '{}' | sha256sum` +* the squashfs digest is the hash of the squashfs file + +The manifest itself is referred to as + +We can pull individual files/artifacts, aka blobs: + +```console +$ oras blob fetch --descriptor \ + localhost:5862/deploy/cluster/zen3/app/1.0@sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db +{ + "mediaType": "application/octet-stream", + "digest": "sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db", + "size": 4096 +} +$ oras blob fetch --output=store.squashfs \ + localhost:5862/deploy/cluster/zen3/app/1.0@sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db +``` + +Note that `blob fetch --descriptor` returns file size, but not file name (`store.squashfs`) +* the file name is in `layers[0]['annotations']['org.opencontainers.image.title']` + +## downloading squashfs + +There are two methods: `oras blob fetch` and `oras pull` + +``` +TAG=v1 +DIGEST=sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db +URL=localhost:5862/deploy/cluster/zen3/app/1.0 + +# blob fetch uses the digest +oras blob fetch --output=store.squashfs $URL@$DIGEST + +# pull uses the digest or tag +oras pull --output=store.squashfs $URL:$TAG +oras pull --output=store.squashfs $URL@$DIGEST +``` + +## downloading meta path + +There is one way: use `oras pull` + +```console +$ TAG=v1 +$ URL=localhost:5862/deploy/cluster/zen3/app/1.0 + +$ oras discover --format=json --artifact-type=uenv/meta localhost:5862/deploy/cluster/zen3/app/1.0:v1 | jq -r '.manifests[0].digest' +sha256:d5d9a6eb9eeb83efffe36f197321cd4b621b2189f066dd7a4b7e9a9e6c61df37 + +$ DIGEST=sha256:d5d9a6eb9eeb83efffe36f197321cd4b621b2189f066dd7a4b7e9a9e6c61df37 +$ oras pull $URL@$DIGEST + +# using image blob fetch downloads something, but I don't know how to turn it into the meta path +oras blob fetch --output meta $URL@$DIGEST +``` + +# Abstraction + +Requirements: + +* squashfs: size, sha, +* url +* meta: sha + + +```cpp +struct manifest { + sha256 digest; + sha256 squashfs_digest; + size_t squashfs_bytes; + optional meta_digest; + std::string respository; + std::string tag; +} +``` + + +Before we used the `uenv-list` service to generate a full database, which we could then search on + +wait now, a manifest is +* a json file +* a description using OCI format of layers and meta + +``` +// complete all information +// - sha of squashfs, sha of meta, full url +manifest fetch_manifest(url, record) { + tag_url = url + record; + mf = oras manifest fetch tag_url + meta = oras discover tag_url + digest = oras resolve tag_url + manifest M { + .digest = ??; + .squashfs = mf[layers][0][digest] + .squashfs_bytes = mf[layers][0][size] + .meta = meta[manifests][0].digest; + .repository = url + record (no tag/digets); + } + +} + +pull_digest(url, sha, path) { + +} + +``` + +uenv image push + TODO check for existing image + oras push --artifact-type=application/x-squashfs squashfs tag + oras attach --artifact-type=uenv/meta tag meta_path + +uenv image pull + TODO check for existing image + oras pull --output=path/store.squashfs manifest.repository@manifest.squashfs_digest + oras pull --output=path/store.squashfs manifest.repository@manifest.meta_digest + +uenv image delete + TODO check for existing image + curl -X Delete ... + +uenv image copy + +# WORKFLOW + +If the rego.supports_search(): + vector registry.match(input_label); + if there is one match then set record +If record is unique + pull + delete + copy + + +``` +struct registry + util::expected + listing(const std::string& nspace) const; + url() const; + supports_search() const; + type() const; + manifest(label) +``` + diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index b72981bd..47b60c14 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -99,7 +99,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, return 1; } - auto src_registry = registry_backend->get_listing(*src_label.nspace); + auto src_registry = registry_backend->listing(*src_label.nspace); if (!src_registry) { term::error("unable to get a listing of the uenv: {}", src_registry.error()); @@ -162,7 +162,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, spdlog::info("destination record: {} {}", dst_record.sha, dst_record); // check whether the destination already exists - auto dst_registry = registry_backend->get_listing(*dst_label.nspace); + auto dst_registry = registry_backend->listing(*dst_label.nspace); if (dst_registry && dst_registry->contains(dst_record)) { if (!args.force) { term::error("the destination already exists - use the --force flag " @@ -172,7 +172,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, term::error("the destination already exists and will be overwritten"); } - const auto rego_url = registry_backend->get_url(); + const auto rego_url = registry_backend->url(); spdlog::debug("registry url: {}", rego_url); if (auto result = oras::copy(rego_url, src_label.nspace.value(), src_record, diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index c745f095..fc9cf833 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -92,7 +92,7 @@ int image_delete([[maybe_unused]] const image_delete_args& args, return 1; } - auto registry = registry_backend->get_listing(nspace); + auto registry = registry_backend->listing(nspace); if (!registry) { term::error("unable to get a listing of the uenv: {}", registry.error()); diff --git a/src/cli/find.cpp b/src/cli/find.cpp index 4db5f28a..29e3a0d6 100644 --- a/src/cli/find.cpp +++ b/src/cli/find.cpp @@ -74,13 +74,13 @@ int image_find([[maybe_unused]] const image_find_args& args, term::error("{}", registry.error()); return 1; } - + if (!registry->supports_search()) { term::error("Registry does not support search functionality"); return 1; } - - auto store = registry->get_listing(nspace); + + auto store = registry->listing(nspace); if (!store) { term::error("unable to get a listing of the uenv: {}", store.error()); return 1; diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index e6b6450d..1dce3497 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -105,10 +105,18 @@ int image_pull([[maybe_unused]] const image_pull_args& args, return 1; } + if (!settings.config.registry) { + term::error("registry is not configured - set it in the config " + "file or provide --registry option"); + return 1; + } + const auto rego_url = settings.config.registry.value(); + spdlog::debug("registry url: {}", rego_url); + uenv_record record{}; if (registry_backend->supports_search()) { - auto registry = registry_backend->get_listing(nspace); + auto registry = registry_backend->listing(nspace); if (!registry) { term::error("unable to get a listing of the uenv: {}", registry.error()); @@ -143,16 +151,25 @@ int image_pull([[maybe_unused]] const image_pull_args& args, } else { spdlog::info("Registry does not support search, proceeding without " "pre-validation"); - // Construct a minimal record from the label for non-searchable - // registries + if (!label.fully_qualified()) { + term::error("the uenv {} to pull must be fully qualified, e.g. " + "'deploy::name/version:tag%system%gh200'", + label); + return 1; + } record.name = label.name.value(); - record.version = label.version.value_or("latest"); - record.tag = label.tag.value_or("latest"); - auto system_name = - site::get_system_name({}, settings.calling_environment); - record.system = label.system.value_or(system_name.value_or("unknown")); - record.uarch = label.uarch.value_or("unknown"); - // Note: sha will be empty, but ORAS operations should still work + record.version = label.version.value(); + record.tag = label.tag.value(); + record.system = label.system.value(); + record.uarch = label.uarch.value(); + // we need to query oras to get the sha + auto sha = oras::pull_sha(rego_url, nspace, record, credentials); + if (!sha) { + term::error("unable to contact registry {}", sha.error().message); + return 1; + } + + record.sha = sha.value(); } spdlog::info("pulling {} {}", record.sha, record); @@ -172,10 +189,11 @@ int image_pull([[maybe_unused]] const image_pull_args& args, auto paths = store->uenv_paths(record.sha); - // acquire a file lock so that only one process can try to pull an image. - // TODO: how do we handle the case where we have many processes waiting, and - // there is a failure (e.g. file system problem), that causes the processes - // to attempt the pull one-after-the-other + // acquire a file lock so that only one process can try to pull an + // image. + // TODO: how do we handle the case where we have many processes waiting, + // and there is a failure (e.g. file system problem), that causes the + // processes to attempt the pull one-after-the-other auto lock = util::make_file_lock(paths.store.string() + ".lock"); bool meta_exists = fs::exists(paths.meta); @@ -202,18 +220,10 @@ int image_pull([[maybe_unused]] const image_pull_args& args, spdlog::debug("pull meta: {}", pull_meta); spdlog::debug("pull sqfs: {}", pull_sqfs); - if (!settings.config.registry) { - term::error("registry is not configured - set it in the config " - "file or provide --registry option"); - return 1; - } - auto rego_url = settings.config.registry.value(); - spdlog::debug("registry url: {}", rego_url); - // the digests returned by oras::discover is a list of artifacts - // that have been "oras attach"ed to our squashfs image. This would - // be empty if no meta data was attached - currently we assume that - // meta data has been attached + // that have been "oras attach"ed to our squashfs image. This + // would be empty if no meta data was attached - currently we + // assume that meta data has been attached auto digests = oras::discover(rego_url, nspace, record, credentials); if (!digests) { @@ -264,7 +274,7 @@ int image_pull([[maybe_unused]] const image_pull_args& args, // download may have been skipped if a squashfs with the same sha has // been downloaded, and this download uses a different label. if (registry_backend->supports_search()) { - auto registry = registry_backend->get_listing(nspace); + auto registry = registry_backend->listing(nspace); if (registry) { const auto remote_matches = registry->query(label); if (remote_matches) { diff --git a/src/cli/push.cpp b/src/cli/push.cpp index 66f6a158..b79412f4 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -92,7 +92,7 @@ int image_push([[maybe_unused]] const image_push_args& args, } if (registry_backend->supports_search()) { - auto registry = registry_backend->get_listing(nspace); + auto registry = registry_backend->listing(nspace); if (!registry) { term::error("unable to get a listing of the uenv: {}", registry.error()); diff --git a/src/cli/uenv.cpp b/src/cli/uenv.cpp index 3de2a262..b28fac87 100644 --- a/src/cli/uenv.cpp +++ b/src/cli/uenv.cpp @@ -39,6 +39,7 @@ int main(int argc, char** argv) { uenv::config_base cli_config; uenv::global_settings settings; bool print_version = false; + std::optional config_file_path; CLI::App cli(fmt::format("uenv {}", UENV_VERSION)); cli.add_flag("-v,--verbose", settings.verbose, "enable verbose output"); @@ -50,6 +51,7 @@ int main(int argc, char** argv) { "enable color output"); cli.add_flag("--version", print_version, "print version"); cli.add_option("--repo", cli_config.repo, "the uenv repository"); + cli.add_option("--config", config_file_path, "path to configuration file"); cli.footer(help_footer); @@ -102,11 +104,17 @@ int main(int argc, char** argv) { // set the configuration according to defaults, cli options and config // files. uenv::config_base user_config; - if (auto x = uenv::load_user_config(settings.calling_environment)) { - user_config = *x; + if (auto cfg = uenv::load_user_config(settings.calling_environment, + config_file_path)) { + user_config = *cfg; + } else { + term::error("error in configuration file:\n {}", cfg.error()); + return 1; } + spdlog::info("user config: {}", user_config); const auto default_config = uenv::default_config(settings.calling_environment); + spdlog::info("default config: {}", default_config); const auto full_config = uenv::merge(cli_config, uenv::merge(user_config, default_config)); if (auto merged_config = uenv::generate_configuration(full_config)) { diff --git a/src/cli/util.cpp b/src/cli/util.cpp index 6c9fb653..7ff5a975 100644 --- a/src/cli/util.cpp +++ b/src/cli/util.cpp @@ -70,8 +70,6 @@ create_registry_from_config(const configuration& config) { case registry_type::site: return site::create_site_registry(); case registry_type::oci: - case registry_type::zot: - case registry_type::ghcr: return create_registry(registry_url, config.registry_type_val); } diff --git a/src/site/site.cpp b/src/site/site.cpp index e033aa4d..a75977b2 100644 --- a/src/site/site.cpp +++ b/src/site/site.cpp @@ -166,11 +166,11 @@ get_credentials(std::optional username, // cscs-specific registry implementation struct cscs_registry { util::expected - get_listing(const std::string& nspace) const { + listing(const std::string& nspace) const { return registry_listing(nspace); } - std::string get_url() const { + std::string url() const { return registry_url(); } @@ -178,9 +178,15 @@ struct cscs_registry { return true; } - uenv::registry_type get_type() const { + uenv::registry_type type() const { return uenv::registry_type::site; } + + util::expected + manifest(const std::string&, const uenv::uenv_label&) const { + // TODO: fill this out + return {}; + } }; uenv::registry create_site_registry() { diff --git a/src/uenv/oras.cpp b/src/uenv/oras.cpp index 850e42a2..ee162af0 100644 --- a/src/uenv/oras.cpp +++ b/src/uenv/oras.cpp @@ -4,6 +4,9 @@ #include #include +#include +#include + #include #include #include @@ -11,6 +14,8 @@ #include #include +#include +#include #include #include #include @@ -20,6 +25,19 @@ namespace uenv { namespace oras { +namespace impl { + +void add_credentials(std::vector& args, + const std::optional token) { + if (token) { + args.push_back("--password"); + args.push_back(token->token); + args.push_back("--username"); + args.push_back(token->username); + } +} +} // namespace impl + using opt_creds = std::optional; // stores the "result" of calling the oras cli tool as an external process. @@ -148,6 +166,25 @@ run_oras_async(std::vector args, return util::run(args, runpath); } +// Wraps `oras discover` +// Oras discover returns a list of manifests that refer to a tag: +// oras discover --format=json --artifact-type=uenv/meta +// localhost:5862/deploy/cluster/zen3/app/1.0:v1 +// { +// "manifests": [ +// { +// ... +// "digest": "sha256:....", +// "artifactType": "uenv/meta", +// ... +// } +// ] +// } +// +// NOTE: we currently restrict the search to 'uenv/meta' artifacts +// +// returns a list of shas in the form "sha256:<64 characters>" +// The list should be of length 0 or 1 util::expected, error> discover(const std::string& registry, const std::string& nspace, const uenv_record& uenv, const opt_creds token) { @@ -157,12 +194,8 @@ discover(const std::string& registry, const std::string& nspace, std::vector args = {"discover", "--format", "json", "--artifact-type", "uenv/meta", address}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); + auto result = run_oras(args); if (result.returncode) { @@ -198,12 +231,8 @@ pull_digest(const std::string& registry, const std::string& nspace, std::vector args{"pull", "--output", destination.string(), address}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); + auto proc = run_oras(args); if (proc.returncode) { @@ -214,6 +243,29 @@ pull_digest(const std::string& registry, const std::string& nspace, return {}; } +util::expected pull_sha(const std::string& registry, + const std::string& nspace, + const uenv_record& uenv, + const opt_creds token) { + auto address = + fmt::format("{}/{}/{}/{}/{}/{}:{}", registry, nspace, uenv.system, + uenv.uarch, uenv.name, uenv.version, uenv.tag); + + spdlog::debug("oras::pull_sha: {}", address); + + std::vector args{"resolve", address}; + impl::add_credentials(args, token); + + auto proc = run_oras(args); + + if (proc.returncode) { + spdlog::error("unable to resolve image sha with oras: {}", proc.stderr); + return util::unexpected{create_error(proc)}; + } + + return uenv::sha256(uenv::parse_oras_sha256(proc.stdout).value()); +} + util::expected pull_tag(const std::string& registry, const std::string& nspace, const uenv_record& uenv, @@ -230,12 +282,8 @@ util::expected pull_tag(const std::string& registry, spdlog::debug("oras::pull_tag: {}", address); std::vector args{"pull", "--concurrency", "10", "--output", destination.string(), address}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); + auto proc = run_oras_async(args); if (!proc) { @@ -305,13 +353,7 @@ util::expected push_tag(const std::string& registry, // Prepare the arguments for the push command std::vector args{"push", "--concurrency", "10"}; - - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); // Add artifact type and annotations args.push_back("--artifact-type"); @@ -399,12 +441,7 @@ util::expected push_meta(const std::string& registry, // Prepare the arguments for the attach command std::vector args{"attach"}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); // Add artifact type and annotations for metadata args.push_back("--artifact-type"); @@ -491,5 +528,37 @@ copy(const std::string& registry, const std::string& src_nspace, return {}; } +util::expected +manifest(const std::string& registry, const uenv_record& uenv, + const std::optional token) { + auto address = fmt::format("{}/{}/{}/{}/{}:{}", registry, uenv.system, + uenv.uarch, uenv.name, uenv.version, uenv.tag); + std::vector args = {"manifest", "fetch", address}; + impl::add_credentials(args, token); + + auto result = run_oras(args); + + if (result.returncode) { + spdlog::error("oras manifest fetch returncode={} stderr='{}'", + result.returncode, result.stderr); + return util::unexpected{create_error(result)}; + } + + std::vector manifests; + using json = nlohmann::json; + uenv::manifest M; + try { + const auto raw = json::parse(result.stdout); + const auto& layer = raw["layers"][0]; + M.squashfs_digest = parse_oras_sha256(layer["digest"]).value(); + M.squashfs_bytes = layer["size"]; + } catch (std::exception& e) { + spdlog::error("error parsing oras manifest fetch output: {}", e.what()); + return util::unexpected(generic_error(e.what())); + } + + return M; +} + } // namespace oras } // namespace uenv diff --git a/src/uenv/oras.h b/src/uenv/oras.h index 2eb5d717..9243e624 100644 --- a/src/uenv/oras.h +++ b/src/uenv/oras.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -60,12 +61,22 @@ push_meta(const std::string& registry, const std::string& nspace, const uenv_label& label, const std::filesystem::path& meta_path, const std::optional token = std::nullopt); +util::expected +pull_sha(const std::string& registry, const std::string& nspace, + const uenv_record& uenv, + const std::optional token = std::nullopt); + util::expected copy(const std::string& registry, const std::string& src_nspace, const uenv_record& src_uenv, const std::string& dst_nspace, const uenv_record& dst_uenv, const std::optional token = std::nullopt); +// combine registry +util::expected +manifest(const std::string& registry, const uenv_record& uenv, + const std::optional token = std::nullopt); + } // namespace oras } // namespace uenv diff --git a/src/uenv/parse.cpp b/src/uenv/parse.cpp index a7fd6111..fe5fc538 100644 --- a/src/uenv/parse.cpp +++ b/src/uenv/parse.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace uenv { @@ -619,4 +620,57 @@ parse_registry_url(const std::string& arg) { return arg; } +// tokens that can appear in names +// names are used for uenv names, versions, tags +bool is_sha_tok(lex::tok t) { + return t == lex::tok::symbol || t == lex::tok::integer; +}; + +// parse the sha from a string of the form: +// sha256:34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15 +util::expected +parse_oras_sha256(const std::string& arg) { + const auto line = util::strip(arg); + + auto L = lex::lexer(line); + + if (L.peek().spelling != "sha") { + return util::unexpected{parse_error{ + L.string(), "oras sha is must start with 'sha256", L.peek()}}; + } + L.next(); + if (L.peek().spelling != "256") { + return util::unexpected{parse_error{ + L.string(), "oras sha is must start with 'sha256", L.peek()}}; + } + L.next(); + if (L.peek().kind != lex::tok::colon) { + return util::unexpected{parse_error{ + L.string(), + fmt::format("unexpected symbol '{}'", L.peek().spelling), + L.peek()}}; + } + L.next(); + + auto result = parse_string(L, "sha256", is_sha_tok); + if (!result) { + return result; + } + + const auto r = *result; + if (L.peek().kind != lex::tok::end) { + return util::unexpected{parse_error{ + L.string(), + fmt::format("unexpected symbol '{}'", L.peek().spelling), + L.peek()}}; + } + if (r.size() != 64) { + return util::unexpected{parse_error{ + L.string(), fmt::format("sha is not 64 characters in length"), + L.peek()}}; + } + + return result; +} + } // namespace uenv diff --git a/src/uenv/parse.h b/src/uenv/parse.h index 7f2c9f4e..950653ab 100644 --- a/src/uenv/parse.h +++ b/src/uenv/parse.h @@ -70,4 +70,7 @@ parse_config_line(const std::string& arg); util::expected parse_registry_url(const std::string& arg); +util::expected +parse_oras_sha256(const std::string& arg); + } // namespace uenv diff --git a/src/uenv/registry.cpp b/src/uenv/registry.cpp index 53ad7236..869525cb 100644 --- a/src/uenv/registry.cpp +++ b/src/uenv/registry.cpp @@ -17,7 +17,7 @@ struct oci_registry { } util::expected - get_listing(const std::string& nspace) const { + listing(const std::string& nspace) const { spdlog::debug("OCI registry does not support listing for namespace: {}", nspace); // Return empty repository - OCI registries generally don't support @@ -25,7 +25,7 @@ struct oci_registry { return uenv::create_repository(); } - std::string get_url() const { + std::string url() const { return url_; } @@ -33,80 +33,23 @@ struct oci_registry { return false; } - registry_type get_type() const { + registry_type type() const { return registry_type::oci; } -}; - -// Zot Registry implementation -struct zot_registry { - private: - std::string url_; - - public: - explicit zot_registry(const std::string& url) : url_(url) { - } - - util::expected - get_listing(const std::string& nspace) const { - spdlog::debug("Zot registry listing for namespace: {}", nspace); - // Zot supports some search capabilities, but for now return empty - // This could be extended to use Zot's GraphQL API - return uenv::create_repository(); - } - std::string get_url() const { - return url_; - } - - bool supports_search() const { - return false; // Could be true if we implement Zot's GraphQL API - } - - registry_type get_type() const { - return registry_type::zot; - } -}; - -// GitHub Container Registry implementation -struct ghcr_registry { - private: - std::string url_; - - public: - explicit ghcr_registry(const std::string& url) : url_(url) { - } - - util::expected - get_listing(const std::string& nspace) const { - spdlog::debug( - "GHCR registry does not support listing for namespace: {}", nspace); - // GHCR doesn't support comprehensive search via OCI APIs - return uenv::create_repository(); - } - - std::string get_url() const { - return url_; - } - - bool supports_search() const { - return false; - } - - registry_type get_type() const { - return registry_type::ghcr; + util::expected + manifest(const std::string&, const uenv::uenv_label&) const { + // TODO: fill this out + return {}; } }; // Factory function implementation registry create_registry(const std::string& url, registry_type type) { + spdlog::debug("creating registry {}::{}", type, url); switch (type) { case registry_type::oci: return registry{oci_registry{url}}; - case registry_type::zot: - return registry{zot_registry{url}}; - case registry_type::ghcr: - return registry{ghcr_registry{url}}; case registry_type::site: // Site-specific implementation will be created in site module spdlog::error("Site registry type should be created via site module"); @@ -119,35 +62,15 @@ registry create_registry(const std::string& url, registry_type type) { // Parse registry type from string util::expected -parse_registry_type(const std::string& type_str) { - if (type_str == "oci") { +parse_registry_type(const std::string& type) { + if (type == "oci") { return registry_type::oci; - } else if (type_str == "zot") { - return registry_type::zot; - } else if (type_str == "ghcr") { - return registry_type::ghcr; - } else if (type_str == "site") { + } else if (type == "site") { return registry_type::site; } else { return util::unexpected(fmt::format( - "Invalid registry type: {}. Valid types are: oci, zot, ghcr, site", - type_str)); - } -} - -// Convert registry type to string -std::string registry_type_to_string(registry_type type) { - switch (type) { - case registry_type::oci: - return "oci"; - case registry_type::zot: - return "zot"; - case registry_type::ghcr: - return "ghcr"; - case registry_type::site: - return "site"; + "Invalid registry type: {}. Valid types are: oci, site", type)); } - return "unknown"; } } // namespace uenv diff --git a/src/uenv/registry.h b/src/uenv/registry.h index a02721bc..e4154543 100644 --- a/src/uenv/registry.h +++ b/src/uenv/registry.h @@ -4,28 +4,42 @@ #include #include +#include + #include +#include #include namespace uenv { +struct manifest { + sha256 digest; + sha256 squashfs_digest; + size_t squashfs_bytes; + std::optional meta_digest; + std::string respository; + std::string tag; +}; + enum class registry_type { - oci, // Generic OCI registry (Docker Hub, etc.) - zot, // Zot registry implementation - ghcr, // GitHub Container Registry - site // Site-specific implementation (JFrog, etc.) + oci, // Generic OCI registry (Docker Hub, etc.) + site // Site-specific implementation (JFrog, etc.) }; // Concept for registry implementations // Any type T that implements these operations can be used as a registry template -concept RegistryImpl = requires(T registry, const std::string& nspace) { +concept RegistryImpl = requires(T registry, const std::string& nspace, + const uenv::uenv_label& label) { { - registry.get_listing(nspace) + registry.listing(nspace) } -> std::convertible_to>; - { registry.get_url() } -> std::convertible_to; + { registry.url() } -> std::convertible_to; { registry.supports_search() } -> std::convertible_to; - { registry.get_type() } -> std::convertible_to; + { registry.type() } -> std::convertible_to; + { + registry.manifest(nspace, label) + } -> std::convertible_to>; }; // Type-erased registry class using value semantics @@ -48,23 +62,25 @@ class registry { // Get listing of images in a namespace // Returns empty repository for registries that don't support search util::expected - get_listing(const std::string& nspace) const { - return impl_->get_listing(nspace); + listing(const std::string& nspace) const { + return impl_->listing(nspace); } - // Get the registry URL - std::string get_url() const { - return impl_->get_url(); + std::string url() const { + return impl_->url(); } - // Whether this registry supports search/listing operations bool supports_search() const { return impl_->supports_search(); } - // Get the registry type - registry_type get_type() const { - return impl_->get_type(); + registry_type type() const { + return impl_->type(); + } + + util::expected + manifest(const std::string& nspace, const uenv_label& label) const { + return impl_->manifest(nspace, label); } private: @@ -72,10 +88,12 @@ class registry { virtual ~interface() = default; virtual std::unique_ptr clone() = 0; virtual util::expected - get_listing(const std::string& nspace) const = 0; - virtual std::string get_url() const = 0; + listing(const std::string& nspace) const = 0; + virtual std::string url() const = 0; virtual bool supports_search() const = 0; - virtual registry_type get_type() const = 0; + virtual registry_type type() const = 0; + virtual util::expected + manifest(const std::string& nspace, const uenv_label& label) const = 0; }; std::unique_ptr impl_; @@ -91,20 +109,26 @@ class registry { } virtual util::expected - get_listing(const std::string& nspace) const override { - return wrapped.get_listing(nspace); + listing(const std::string& nspace) const override { + return wrapped.listing(nspace); } - virtual std::string get_url() const override { - return wrapped.get_url(); + virtual std::string url() const override { + return wrapped.url(); } virtual bool supports_search() const override { return wrapped.supports_search(); } - virtual registry_type get_type() const override { - return wrapped.get_type(); + virtual registry_type type() const override { + return wrapped.type(); + } + + virtual util::expected + manifest(const std::string& nspace, + const uenv_label& label) const override { + return wrapped.manifest(nspace, label); } T wrapped; @@ -118,7 +142,26 @@ registry create_registry(const std::string& url, registry_type type); util::expected parse_registry_type(const std::string& type_str); -// Convert registry type to string -std::string registry_type_to_string(registry_type type); - } // namespace uenv + +template <> class fmt::formatter { + public: + // parse format specification and store it: + constexpr auto parse(format_parse_context& ctx) { + return ctx.end(); + } + // format a value using stored specification: + template + constexpr auto format(uenv::registry_type const& type, + FmtContext& ctx) const { + using uenv::registry_type; + + switch (type) { + case registry_type::oci: + return fmt::format_to(ctx.out(), "oci"); + case registry_type::site: + return fmt::format_to(ctx.out(), "site"); + } + return fmt::format_to(ctx.out(), "unknown"); + } +}; diff --git a/src/uenv/settings.cpp b/src/uenv/settings.cpp index 382af5ec..5102c305 100644 --- a/src/uenv/settings.cpp +++ b/src/uenv/settings.cpp @@ -53,7 +53,7 @@ config_base merge(const config_base& lhs, const config_base& rhs) { config_base default_config(const envvars::state& env) { return { .repo = default_repo_path(env), - .registry = "jfrog.svc.cscs.ch/uenv", // Default site registry URL + .registry = "jfrog.svc.cscs.ch/uenv", // Default site registry URL .registry_type = "site", .color = color::default_color(env), }; @@ -104,44 +104,57 @@ read_config_file(const std::filesystem::path& path, } util::expected -load_user_config(const envvars::state& calling_env) { +load_user_config(const envvars::state& calling_env, + const std::optional& config_path) { namespace fs = std::filesystem; - auto home_env = calling_env.get("HOME"); - auto xdg_env = calling_env.get("XDG_CONFIG_HOME"); - // return an empty config if no configuration path can be determined - if (!home_env && !xdg_env) { - spdlog::warn("unable to find default configuration location, neither " - "HOME nor XDG_CONFIG_HOME are defined."); - return config_base{}; - } - const auto config_path = - xdg_env ? (fs::path(xdg_env.value()) / "uenv") - : (fs::path(home_env.value()) / ".config/uenv"); - const auto config_file = config_path / "config"; - - auto create_config_file = [](const auto& path) { - auto fid = std::ofstream(path); - fid << config_file_default << std::endl; - }; - if (!fs::exists(config_path)) { - spdlog::info("creating configuration path {}", config_path); - std::error_code ec; - fs::create_directories(config_path, ec); - if (ec) { - spdlog::error("unable to create config path: {}", ec.message()); + fs::path config_file; + + // if a custom config path is provided, use it directly + if (config_path) { + spdlog::info("using custom configuration file {}", + config_path->string()); + + if (!(fs::exists(*config_path) && fs::is_regular_file(*config_path))) { + return util::unexpected(fmt::format( + "custom configuration file '{}' is not a regular file", + config_path->string())); + } + + config_file = config_path.value(); + } else { + auto home_env = calling_env.get("HOME"); + auto xdg_env = calling_env.get("XDG_CONFIG_HOME"); + // return an empty config if no configuration path can be determined + if (!home_env && !xdg_env) { + spdlog::warn( + "unable to find default configuration location, neither " + "HOME nor XDG_CONFIG_HOME are defined."); + return config_base{}; + } + const auto config_dir = + xdg_env ? (fs::path(xdg_env.value()) / "uenv") + : (fs::path(home_env.value()) / ".config/uenv"); + config_file = config_dir / "config"; + + if (!fs::exists(config_dir)) { + spdlog::info("creating configuration path {}", config_dir); + std::error_code ec; + fs::create_directories(config_dir, ec); + if (ec) { + spdlog::error("unable to create config path: {}", ec.message()); + return config_base{}; + } + } + if (!fs::exists(config_file)) { + spdlog::info("creating configuration file {}", config_file); + auto fid = std::ofstream(config_file); + fid << config_file_default << std::endl; return config_base{}; } - spdlog::info("creating configuration file {}", config_file); - create_config_file(config_file); - return config_base{}; - } else if (!fs::exists(config_file)) { - spdlog::info("creating configuration file {}", config_file); - create_config_file(config_file); - return config_base{}; } - spdlog::info("opening configuration file {}", config_file); + spdlog::info("parsing configuration file {}", config_file); auto result = impl::read_config_file(config_file, calling_env); if (!result) { @@ -149,7 +162,7 @@ load_user_config(const envvars::state& calling_env) { "error opening '{}': {}", config_file.string(), result.error())}; } - return *result; + return result.value(); } namespace impl { @@ -194,6 +207,8 @@ read_config_file(const std::filesystem::path& path, fid.close(); + spdlog::debug("finished parsing file"); + // build a config from the key value config_base config; for (auto [key, value] : settings) { @@ -222,7 +237,7 @@ read_config_file(const std::filesystem::path& path, fmt::format("invalid reguistry url '{}={}': {}", key, value, p.error().message())); } - } else if (key == "registry_type") { + } else if (key == "registry-type") { config.registry_type = value; } else { return util::unexpected( diff --git a/src/uenv/settings.h b/src/uenv/settings.h index b6fe2ff5..0b67fcf7 100644 --- a/src/uenv/settings.h +++ b/src/uenv/settings.h @@ -4,6 +4,8 @@ #include #include +#include + #include #include #include @@ -29,8 +31,10 @@ struct config_line { // read configuration from the user configuration file // the location of the config file is determined using XDG_CONFIG_HOME or HOME -util::expected -load_user_config(const envvars::state&); +// if config_path is provided, use it directly instead of the default locations +util::expected load_user_config( + const envvars::state&, + const std::optional& config_path = std::nullopt); // get the default configuration config_base default_config(const envvars::state& calling_env); @@ -49,3 +53,19 @@ util::expected generate_configuration(const config_base& base); } // namespace uenv + +template <> class fmt::formatter { + public: + // parse format specification and store it: + constexpr auto parse(format_parse_context& ctx) { + return ctx.end(); + } + // format a value using stored specification: + template + constexpr auto format(uenv::config_base const& cfg, FmtContext& ctx) const { + return fmt::format_to( + ctx.out(), "[repo: {}, registry: {}, registry_type: {}, color: {}]", + cfg.repo.value_or("()"), cfg.registry.value_or("()"), + cfg.registry_type.value_or("()"), cfg.color.value_or("()")); + } +}; diff --git a/test/integration/cli.bats b/test/integration/cli.bats index a5f43d27..4d0ffc00 100644 --- a/test/integration/cli.bats +++ b/test/integration/cli.bats @@ -23,13 +23,12 @@ function setup() { # remove the bash function uenv, if an older version of uenv is installed on # the system unset -f uenv - - # Set up test registry - setup_test_registry } function teardown() { - teardown_test_registry + : + # this will tear down the test registry if it was started inside the test + #teardown_test_registry } @test "noargs" { @@ -476,6 +475,9 @@ EOF skip "Docker not available for registry tests" fi + # Set up test registry + setup_test_registry + # Verify registry is accessible run curl -s "http://localhost:$REGISTRY_PORT/v2/" assert_success @@ -496,39 +498,40 @@ EOF refute_output --partial "$container_name" } -@test "zot registry push pull" { +@test "oci registry push pull" { + # Set up test registry + setup_test_registry + # Skip if Docker is not available if ! registry_is_available; then skip "Docker not available for registry tests" fi + local work=$(mktemp -d $TMP/push-test-XXXXXX) + # Create a test repository for the source image - local src_repo=$(mktemp -d $TMP/push-test-XXXXXX) - run uenv repo create $src_repo + local repo=$work/repo + run uenv repo create $repo assert_success - # Add an app image to the source repo - run uenv --repo=$src_repo image add test-app/1.0:v1@test%zen3 $SQFS_LIB/apptool/standalone/app42.squashfs - assert_success + # create a config file that points to the working repo and registry + local config=$work/config + echo "repo = $repo" > $config + echo "registry = $REGISTRY_URL" >> $config + echo "registry-type = oci" >> $config - # Push the image to the Zot registry - run uenv --repo=$src_repo --registry=$REGISTRY_URL --registry-type=zot \ - image push test-app/1.0:v1@test%zen3 test::test-app/1.0:v1@test%zen3 + # Push the image to the oci registry + sqfs_path=$SQFS_LIB/apptool/standalone/app42.squashfs + run uenv -vv --config=$config image push $sqfs_path deploy::app/1.0:v1@cluster%zen3 assert_success assert_output --partial "successfully pushed" - # Create a separate repository for pulling - local dst_repo=$(mktemp -d $TMP/pull-test-XXXXXX) - run uenv repo create $dst_repo - assert_success - - # Pull the image from the Zot registry - run uenv --repo=$dst_repo --registry=$REGISTRY_URL --registry-type=zot \ - image pull test::test-app/1.0:v1@test%zen3 + # Pull the image from the oci registry + run uenv -vvv --config=$config image pull deploy::app/1.0:v1@cluster%zen3 assert_success # Verify the image was pulled successfully - run uenv --repo=$dst_repo image ls test-app --no-header + run uenv --config=$config image ls test-app --no-header assert_success assert_output --partial "test-app/1.0:v1" assert_output --partial "zen3" diff --git a/test/unit/parse.cpp b/test/unit/parse.cpp index b34481e8..4706aecc 100644 --- a/test/unit/parse.cpp +++ b/test/unit/parse.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -362,11 +361,7 @@ TEST_CASE("parse registry entry", "[parse]") { "build/eiger/zen2/prgenv-gnu/24.11/1529952520", }) { auto r = uenv::parse_registry_entry(s); - if (!r) { - fmt::println("{}", r.error().message()); - } else { - REQUIRE(r); - } + REQUIRE(r); } } @@ -473,3 +468,45 @@ TEST_CASE("config_line", "[parse]") { REQUIRE(!result); } } + +TEST_CASE("oras_sha", "[parse]") { + { + auto sha = uenv::parse_oras_sha256( + "sha256:" + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15"); + REQUIRE(sha); + REQUIRE( + sha.value() == + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15"); + } + { + auto sha = uenv::parse_oras_sha256( + "sha256:" + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c1"); + REQUIRE(!sha); + } + { + auto sha = uenv::parse_oras_sha256( + "sha255:" + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15"); + REQUIRE(!sha); + } + { + auto sha = uenv::parse_oras_sha256( + "sha256@" + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15"); + REQUIRE(!sha); + } + { + auto sha = uenv::parse_oras_sha256("sha256:" + "34c77667fa06e4c73bf98e357a8823b7eb0" + "a2a38a84b22d03fed5b45387f9c15-"); + REQUIRE(!sha); + } + { + auto sha = uenv::parse_oras_sha256("sha256:" + "34c77667fa06e4c73bf98e357a8823b7eb0" + "a2a-38a84b22d03fed5b45387f9c15"); + REQUIRE(!sha); + } +}