diff --git a/.gitignore b/.gitignore index ed2558a1..4cdda01e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ install build pyenv + +# npm artifacts installed by tools +package*.json +node_modules diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..5d45759f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,110 @@ +# Registry Abstraction Implementation Status + +## Context +Refactoring hardcoded site-specific JFrog registry to support multiple registry types (OCI, Zot, GHCR, site-specific). Goal is to allow users to configure different registries while maintaining backward compatibility. + +## Key Understanding +- **Repository**: Local disk storage (SQLite + subdirectories) for downloaded images +- **Registry**: Remote OCI container registry (JFrog, GHCR, Docker Hub, etc.) +- **Repository data structure**: Also used as in-memory container for registry search results (consistent query interface) + +## Completed Work ✅ + +### 1. Core Abstraction (`src/uenv/registry.h` + `src/uenv/registry.cpp`) +- Created `registry_backend` abstract base class +- Implemented `registry_type` enum (oci, zot, ghcr, site) +- Added factory function `create_registry(url, type)` +- OCI/Zot/GHCR implementations return empty repositories (no search support) +- Added `supports_search()` capability flag + +### 2. Site Integration (`src/site/site.h` + `src/site/site.cpp`) +- Added `jfrog_registry` class wrapping existing `registry_listing()` function +- Added `create_site_registry()` factory function +- Preserves existing JFrog custom API behavior + +### 3. Configuration (`src/uenv/settings.h` + `src/uenv/settings.cpp`) +- Added `registry_type` field to `config_base` and `configuration` +- Added parsing for registry_type config parameter +- Default type is "site" for backward compatibility + +### 4. CLI Utilities (`src/cli/util.h` + `src/cli/util.cpp`) +- Added `create_registry_from_config()` helper function +- Handles registry backend creation based on configuration + +### 5. Build System (`meson.build`) +- Added `src/uenv/registry.cpp` to lib_src + +### 6. Complete CLI Updates ✅ +- Updated `src/cli/copy.cpp` to use new registry abstraction +- Updated `src/cli/delete.cpp` to use registry abstraction with graceful degradation +- Updated `src/cli/pull.cpp` to use registry abstraction with graceful degradation +- Updated `src/cli/push.cpp` to use registry abstraction with graceful degradation +- ✅ All CLI files now use the registry abstraction +- ✅ Code compiles and tests pass + +### 7. Type Erasure Refactoring ✅ +- Refactored registry from abstract base class to type-erased value class +- Used concept-based approach similar to `help::item` pattern +- Registry implementations are now structs that satisfy `RegistryImpl` concept +- Registry objects can be stored by value, copied, and moved +- Updated all CLI utilities to use value-based API instead of pointer-based +- ✅ All code compiles and tests pass + +## Remaining Work 🚧 + +### 2. Implement Graceful Degradation Pattern ✅ +All CLI commands now implement graceful degradation: +- **delete.cpp**: Requires search support, fails gracefully if not available +- **pull.cpp**: Constructs minimal record for non-searchable registries +- **push.cpp**: Skips existence check for non-searchable registries with warning +- **copy.cpp**: Uses search for validation when available + +Pattern implemented: +```cpp +auto registry_backend = create_registry_from_config(settings.config); +if (registry_backend->supports_search()) { + // Existing logic: validate via listing first + auto listing = registry_backend->get_listing(nspace); + // ... validation, disambiguation, etc. +} else { + // Graceful degradation: proceed without validation + spdlog::info("Registry does not support search, proceeding without pre-validation"); + // ... direct ORAS operations with proper error handling +} +``` + +### 3. Enhance ORAS Error Handling +Ensure ORAS wrapper functions return well-structured errors: +- "Image not found" errors +- "Network/registry unreachable" errors +- "Authentication failed" errors +- "Invalid image format" errors + +### 4. Update Error Messages +- With search: "No image found matching 'X' in registry listing" +- Without search: "Failed to pull image 'X': image not found in registry" + +### 5. Testing +- Test searchable registry (JFrog): Existing behavior preserved +- Test non-searchable registry: Graceful operation with proper errors + +## Configuration Examples +``` +# Site-specific (current default) +registry = jfrog.svc.cscs.ch/uenv +registry_type = site + +# GitHub Container Registry +registry = ghcr.io +registry_type = ghcr + +# Generic OCI registry +registry = docker.io +registry_type = oci +``` + +## Next Steps +1. Complete CLI file updates with supports_search() branching +2. Enhance ORAS error handling +3. Test with different registry types +4. Update documentation \ No newline at end of file diff --git a/meson.build b/meson.build index 4f4cd65d..a09c0357 100644 --- a/meson.build +++ b/meson.build @@ -46,6 +46,7 @@ lib_src = [ 'src/uenv/oras.cpp', 'src/uenv/parse.cpp', 'src/uenv/print.cpp', + 'src/uenv/registry.cpp', 'src/uenv/repository.cpp', 'src/uenv/settings.cpp', 'src/uenv/uenv.cpp', diff --git a/oras.md b/oras.md new file mode 100644 index 00000000..85da136e --- /dev/null +++ b/oras.md @@ -0,0 +1,184 @@ + +The *manifest* for a uenv squashfs is a JSON file that can be obtained using `oras manifest fetch rego/tag` + +* there is always the same empty `config` field + * `e30=` is an empty JSON object `{}` in base64 encoding +* there is a single layer: the squashfs file +* there is a single annotation with the time of creation + +```console +$ oras manifest fetch localhost:5862/deploy/cluster/zen3/app/1.0:v1 | jq . +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "artifactType": "application/x-squashfs", + "config": { + "mediaType": "application/vnd.oci.empty.v1+json", + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "size": 2, + "data": "e30=" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "digest": "sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db", + "size": 4096, + "annotations": { + "org.opencontainers.image.title": "app42.squashfs" + } + } + ], + "annotations": { + "org.opencontainers.image.created": "2025-06-17T08:32:08Z" + } +} + +``` + +Every file (squashfs, tar ball of meta path, manifest.json, etc) in a registry has a *digest* of the form `sha256:<64 character sha>`. + +* the config json has a digest that is `printf '{}' | sha256sum` +* the squashfs digest is the hash of the squashfs file + +The manifest itself is referred to as + +We can pull individual files/artifacts, aka blobs: + +```console +$ oras blob fetch --descriptor \ + localhost:5862/deploy/cluster/zen3/app/1.0@sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db +{ + "mediaType": "application/octet-stream", + "digest": "sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db", + "size": 4096 +} +$ oras blob fetch --output=store.squashfs \ + localhost:5862/deploy/cluster/zen3/app/1.0@sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db +``` + +Note that `blob fetch --descriptor` returns file size, but not file name (`store.squashfs`) +* the file name is in `layers[0]['annotations']['org.opencontainers.image.title']` + +## downloading squashfs + +There are two methods: `oras blob fetch` and `oras pull` + +``` +TAG=v1 +DIGEST=sha256:4f9aa7ee3a5a056c3742119a69b2aa6eafefa46e571ce3f377f67aebdf43c2db +URL=localhost:5862/deploy/cluster/zen3/app/1.0 + +# blob fetch uses the digest +oras blob fetch --output=store.squashfs $URL@$DIGEST + +# pull uses the digest or tag +oras pull --output=store.squashfs $URL:$TAG +oras pull --output=store.squashfs $URL@$DIGEST +``` + +## downloading meta path + +There is one way: use `oras pull` + +```console +$ TAG=v1 +$ URL=localhost:5862/deploy/cluster/zen3/app/1.0 + +$ oras discover --format=json --artifact-type=uenv/meta localhost:5862/deploy/cluster/zen3/app/1.0:v1 | jq -r '.manifests[0].digest' +sha256:d5d9a6eb9eeb83efffe36f197321cd4b621b2189f066dd7a4b7e9a9e6c61df37 + +$ DIGEST=sha256:d5d9a6eb9eeb83efffe36f197321cd4b621b2189f066dd7a4b7e9a9e6c61df37 +$ oras pull $URL@$DIGEST + +# using image blob fetch downloads something, but I don't know how to turn it into the meta path +oras blob fetch --output meta $URL@$DIGEST +``` + +# Abstraction + +Requirements: + +* squashfs: size, sha, +* url +* meta: sha + + +```cpp +struct manifest { + sha256 digest; + sha256 squashfs_digest; + size_t squashfs_bytes; + optional meta_digest; + std::string respository; + std::string tag; +} +``` + + +Before we used the `uenv-list` service to generate a full database, which we could then search on + +wait now, a manifest is +* a json file +* a description using OCI format of layers and meta + +``` +// complete all information +// - sha of squashfs, sha of meta, full url +manifest fetch_manifest(url, record) { + tag_url = url + record; + mf = oras manifest fetch tag_url + meta = oras discover tag_url + digest = oras resolve tag_url + manifest M { + .digest = ??; + .squashfs = mf[layers][0][digest] + .squashfs_bytes = mf[layers][0][size] + .meta = meta[manifests][0].digest; + .repository = url + record (no tag/digets); + } + +} + +pull_digest(url, sha, path) { + +} + +``` + +uenv image push + TODO check for existing image + oras push --artifact-type=application/x-squashfs squashfs tag + oras attach --artifact-type=uenv/meta tag meta_path + +uenv image pull + TODO check for existing image + oras pull --output=path/store.squashfs manifest.repository@manifest.squashfs_digest + oras pull --output=path/store.squashfs manifest.repository@manifest.meta_digest + +uenv image delete + TODO check for existing image + curl -X Delete ... + +uenv image copy + +# WORKFLOW + +If the rego.supports_search(): + vector registry.match(input_label); + if there is one match then set record +If record is unique + pull + delete + copy + + +``` +struct registry + util::expected + listing(const std::string& nspace) const; + url() const; + supports_search() const; + type() const; + manifest(label) +``` + diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index 352db24f..47b60c14 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,7 @@ #include "copy.h" #include "help.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -91,9 +93,15 @@ int image_copy([[maybe_unused]] const image_copy_args& args, return 1; } - auto src_registry = site::registry_listing(*src_label.nspace); + auto registry_backend = create_registry_from_config(settings.config); + if (!registry_backend) { + term::error("{}", registry_backend.error()); + return 1; + } + + auto src_registry = registry_backend->listing(*src_label.nspace); if (!src_registry) { - term::error("unable to get a listing of the uenv", + term::error("unable to get a listing of the uenv: {}", src_registry.error()); return 1; } @@ -154,7 +162,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, spdlog::info("destination record: {} {}", dst_record.sha, dst_record); // check whether the destination already exists - auto dst_registry = site::registry_listing(*dst_label.nspace); + auto dst_registry = registry_backend->listing(*dst_label.nspace); if (dst_registry && dst_registry->contains(dst_record)) { if (!args.force) { term::error("the destination already exists - use the --force flag " @@ -164,7 +172,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, term::error("the destination already exists and will be overwritten"); } - const auto rego_url = site::registry_url(); + const auto rego_url = registry_backend->url(); spdlog::debug("registry url: {}", rego_url); if (auto result = oras::copy(rego_url, src_label.nspace.value(), src_record, diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index 40e7fa16..fc9cf833 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include "delete.h" #include "help.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -78,9 +80,22 @@ int image_delete([[maybe_unused]] const image_delete_args& args, } spdlog::debug("requested to delete {}::{}", nspace, label); - auto registry = site::registry_listing(nspace); + auto registry_backend = create_registry_from_config(settings.config); + if (!registry_backend) { + term::error("{}", registry_backend.error()); + return 1; + } + + if (!registry_backend->supports_search()) { + term::error("Registry does not support search - cannot validate uenv " + "for deletion"); + return 1; + } + + auto registry = registry_backend->listing(nspace); if (!registry) { - term::error("unable to get a listing of the uenv", registry.error()); + term::error("unable to get a listing of the uenv: {}", + registry.error()); return 1; } @@ -95,7 +110,7 @@ int image_delete([[maybe_unused]] const image_delete_args& args, using enum help::block::admonition; term::error("no uenv found that matches '{}'\n\n{}", args.uenv_description, - help::block(info, "try searching for the uenv to copy " + help::block(info, "try searching for the uenv to delete " "first using 'uenv image find'")); return 1; } else if (!matches->unique_sha()) { @@ -107,8 +122,14 @@ int image_delete([[maybe_unused]] const image_delete_args& args, return 1; } - const auto rego_url = site::registry_url(); + if (!settings.config.registry) { + term::error("registry is not configured - set it in the config file or " + "provide --registry option"); + return 1; + } + const auto rego_url = settings.config.registry.value(); spdlog::debug("registry url: {}", rego_url); + for (auto& record : *matches) { auto url = fmt::format( "https://jfrog.svc.cscs.ch/artifactory/uenv/{}/{}/{}/{}/{}/{}", diff --git a/src/cli/find.cpp b/src/cli/find.cpp index 87d9b66a..29e3a0d6 100644 --- a/src/cli/find.cpp +++ b/src/cli/find.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include "find.h" #include "help.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -67,9 +69,20 @@ int image_find([[maybe_unused]] const image_find_args& args, site::get_system_name(label.system, settings.calling_environment); spdlog::info("image_find: {}::{}", nspace, label); - auto store = site::registry_listing(nspace); + auto registry = create_registry_from_config(settings.config); + if (!registry) { + term::error("{}", registry.error()); + return 1; + } + + if (!registry->supports_search()) { + term::error("Registry does not support search functionality"); + return 1; + } + + auto store = registry->listing(nspace); if (!store) { - term::error("unable to get a listing of the uenv", store.error()); + term::error("unable to get a listing of the uenv: {}", store.error()); return 1; } diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index d1c75b5a..1dce3497 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include "help.h" #include "pull.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -97,37 +99,78 @@ int image_pull([[maybe_unused]] const image_pull_args& args, spdlog::info("image_pull: {}::{}", nspace, label); - auto registry = site::registry_listing(nspace); - if (!registry) { - term::error("unable to get a listing of the uenv", registry.error()); + auto registry_backend = create_registry_from_config(settings.config); + if (!registry_backend) { + term::error("{}", registry_backend.error()); return 1; } - // search db for matching records - const auto remote_matches = registry->query(label); - if (!remote_matches) { - term::error("invalid search term: {}", registry.error()); - return 1; - } - // check that there is one record with a unique sha - if (remote_matches->empty()) { - using enum help::block::admonition; - term::error("no uenv found that matches '{}'\n\n{}", - args.uenv_description, - help::block(info, "try searching for the uenv to pull " - "first using 'uenv image find'")); - return 1; - } else if (!remote_matches->unique_sha()) { - std::string errmsg = - fmt::format("more than one uenv found that matches '{}':\n", - args.uenv_description); - errmsg += format_record_set(*remote_matches); - term::error("{}", errmsg); + if (!settings.config.registry) { + term::error("registry is not configured - set it in the config " + "file or provide --registry option"); return 1; } + const auto rego_url = settings.config.registry.value(); + spdlog::debug("registry url: {}", rego_url); + + uenv_record record{}; + + if (registry_backend->supports_search()) { + auto registry = registry_backend->listing(nspace); + if (!registry) { + term::error("unable to get a listing of the uenv: {}", + registry.error()); + return 1; + } + + // search db for matching records + const auto remote_matches = registry->query(label); + if (!remote_matches) { + term::error("invalid search term: {}", registry.error()); + return 1; + } + // check that there is one record with a unique sha + if (remote_matches->empty()) { + using enum help::block::admonition; + term::error("no uenv found that matches '{}'\n\n{}", + args.uenv_description, + help::block(info, "try searching for the uenv to pull " + "first using 'uenv image find'")); + return 1; + } else if (!remote_matches->unique_sha()) { + std::string errmsg = + fmt::format("more than one uenv found that matches '{}':\n", + args.uenv_description); + errmsg += format_record_set(*remote_matches); + term::error("{}", errmsg); + return 1; + } - // pick a record to use for pulling - const auto record = *(remote_matches->begin()); + // pick a record to use for pulling + record = *(remote_matches->begin()); + } else { + spdlog::info("Registry does not support search, proceeding without " + "pre-validation"); + if (!label.fully_qualified()) { + term::error("the uenv {} to pull must be fully qualified, e.g. " + "'deploy::name/version:tag%system%gh200'", + label); + return 1; + } + record.name = label.name.value(); + record.version = label.version.value(); + record.tag = label.tag.value(); + record.system = label.system.value(); + record.uarch = label.uarch.value(); + // we need to query oras to get the sha + auto sha = oras::pull_sha(rego_url, nspace, record, credentials); + if (!sha) { + term::error("unable to contact registry {}", sha.error().message); + return 1; + } + + record.sha = sha.value(); + } spdlog::info("pulling {} {}", record.sha, record); // require that a valid repo has been provided @@ -146,10 +189,11 @@ int image_pull([[maybe_unused]] const image_pull_args& args, auto paths = store->uenv_paths(record.sha); - // acquire a file lock so that only one process can try to pull an image. - // TODO: how do we handle the case where we have many processes waiting, and - // there is a failure (e.g. file system problem), that causes the processes - // to attempt the pull one-after-the-other + // acquire a file lock so that only one process can try to pull an + // image. + // TODO: how do we handle the case where we have many processes waiting, + // and there is a failure (e.g. file system problem), that causes the + // processes to attempt the pull one-after-the-other auto lock = util::make_file_lock(paths.store.string() + ".lock"); bool meta_exists = fs::exists(paths.meta); @@ -176,13 +220,10 @@ int image_pull([[maybe_unused]] const image_pull_args& args, spdlog::debug("pull meta: {}", pull_meta); spdlog::debug("pull sqfs: {}", pull_sqfs); - auto rego_url = site::registry_url(); - spdlog::debug("registry url: {}", rego_url); - // the digests returned by oras::discover is a list of artifacts - // that have been "oras attach"ed to our squashfs image. This would - // be empty if no meta data was attached - currently we assume that - // meta data has been attached + // that have been "oras attach"ed to our squashfs image. This + // would be empty if no meta data was attached - currently we + // assume that meta data has been attached auto digests = oras::discover(rego_url, nspace, record, credentials); if (!digests) { @@ -232,15 +273,34 @@ int image_pull([[maybe_unused]] const image_pull_args& args, // add the label to the repo, even if there was no download. // download may have been skipped if a squashfs with the same sha has // been downloaded, and this download uses a different label. - for (auto& r : *remote_matches) { - bool exists = in_repo({.name = r.name, - .version = r.version, - .tag = r.tag, - .system = r.system, - .uarch = r.uarch}); + if (registry_backend->supports_search()) { + auto registry = registry_backend->listing(nspace); + if (registry) { + const auto remote_matches = registry->query(label); + if (remote_matches) { + for (auto& r : *remote_matches) { + bool exists = in_repo({.name = r.name, + .version = r.version, + .tag = r.tag, + .system = r.system, + .uarch = r.uarch}); + if (!exists) { + term::msg("updating {}", r); + store->add(r); + } + } + } + } + } else { + // For non-searchable registries, add the constructed record + bool exists = in_repo({.name = record.name, + .version = record.version, + .tag = record.tag, + .system = record.system, + .uarch = record.uarch}); if (!exists) { - term::msg("updating {}", r); - store->add(r); + term::msg("updating {}", record); + store->add(record); } } diff --git a/src/cli/push.cpp b/src/cli/push.cpp index 0b4d6604..b79412f4 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -83,30 +84,49 @@ int image_push([[maybe_unused]] const image_push_args& args, dst_label.label); const auto nspace = dst_label.nspace.value(); - auto registry = site::registry_listing(nspace); - if (!registry) { - term::error("unable to get a listing of the uenv", registry.error()); - return 1; - } - // check whether an image that matches the target label is already - // in the registry. - const auto remote_matches = registry->query(dst_label.label); - if (!remote_matches) { - term::error("invalid search term: {}", registry.error()); + auto registry_backend = create_registry_from_config(settings.config); + if (!registry_backend) { + term::error("{}", registry_backend.error()); return 1; } - if (!remote_matches->empty() && !args.force) { - using enum help::block::admonition; - term::error( - "a uenv that matches '{}' is already in the registry\n\n{}", - args.dest, - help::block(info, - "use the --force flag if you want to overwrite it ")); - return 1; - } else if (!remote_matches->empty() && args.force) { - spdlog::info("{} already exists and will be overwritten", args.dest); - term::msg("the destination already exists and will be overwritten"); + + if (registry_backend->supports_search()) { + auto registry = registry_backend->listing(nspace); + if (!registry) { + term::error("unable to get a listing of the uenv: {}", + registry.error()); + return 1; + } + + // check whether an image that matches the target label is already + // in the registry. + const auto remote_matches = registry->query(dst_label.label); + if (!remote_matches) { + term::error("invalid search term: {}", registry.error()); + return 1; + } + if (!remote_matches->empty() && !args.force) { + using enum help::block::admonition; + term::error( + "a uenv that matches '{}' is already in the registry\n\n{}", + args.dest, + help::block( + info, "use the --force flag if you want to overwrite it ")); + return 1; + } else if (!remote_matches->empty() && args.force) { + spdlog::info("{} already exists and will be overwritten", + args.dest); + term::msg("the destination already exists and will be overwritten"); + } + } else { + spdlog::info("Registry does not support search, proceeding without " + "pre-validation"); + if (!args.force) { + term::warn( + "Registry does not support search - cannot check for existing " + "images. Consider using --force if overwriting is acceptable."); + } } // validate the source squashfs file @@ -118,7 +138,12 @@ int image_push([[maybe_unused]] const image_push_args& args, spdlog::info("image_push: squashfs {}", sqfs.value()); try { - auto rego_url = site::registry_url(); + if (!settings.config.registry) { + term::error("registry is not configured - set it in the config " + "file or provide --registry option"); + return 1; + } + auto rego_url = settings.config.registry.value(); spdlog::debug("registry url: {}", rego_url); // Push the SquashFS image diff --git a/src/cli/uenv.cpp b/src/cli/uenv.cpp index 3de2a262..b28fac87 100644 --- a/src/cli/uenv.cpp +++ b/src/cli/uenv.cpp @@ -39,6 +39,7 @@ int main(int argc, char** argv) { uenv::config_base cli_config; uenv::global_settings settings; bool print_version = false; + std::optional config_file_path; CLI::App cli(fmt::format("uenv {}", UENV_VERSION)); cli.add_flag("-v,--verbose", settings.verbose, "enable verbose output"); @@ -50,6 +51,7 @@ int main(int argc, char** argv) { "enable color output"); cli.add_flag("--version", print_version, "print version"); cli.add_option("--repo", cli_config.repo, "the uenv repository"); + cli.add_option("--config", config_file_path, "path to configuration file"); cli.footer(help_footer); @@ -102,11 +104,17 @@ int main(int argc, char** argv) { // set the configuration according to defaults, cli options and config // files. uenv::config_base user_config; - if (auto x = uenv::load_user_config(settings.calling_environment)) { - user_config = *x; + if (auto cfg = uenv::load_user_config(settings.calling_environment, + config_file_path)) { + user_config = *cfg; + } else { + term::error("error in configuration file:\n {}", cfg.error()); + return 1; } + spdlog::info("user config: {}", user_config); const auto default_config = uenv::default_config(settings.calling_environment); + spdlog::info("default config: {}", default_config); const auto full_config = uenv::merge(cli_config, uenv::merge(user_config, default_config)); if (auto merged_config = uenv::generate_configuration(full_config)) { diff --git a/src/cli/util.cpp b/src/cli/util.cpp index ca18d680..7ff5a975 100644 --- a/src/cli/util.cpp +++ b/src/cli/util.cpp @@ -4,12 +4,13 @@ #include #include +#include +#include +#include #include #include #include -#include - #include "util.h" namespace uenv { @@ -56,4 +57,23 @@ validate_squashfs_image(const std::string& path) { return img; } +util::expected +create_registry_from_config(const configuration& config) { + if (!config.registry) { + return util::unexpected("registry is not configured - set it in the " + "config file or provide --registry option"); + } + + auto registry_url = config.registry.value().string(); + + switch (config.registry_type_val) { + case registry_type::site: + return site::create_site_registry(); + case registry_type::oci: + return create_registry(registry_url, config.registry_type_val); + } + + return util::unexpected("unknown registry type"); +} + } // namespace uenv diff --git a/src/cli/util.h b/src/cli/util.h index 89f32f2c..7210f267 100644 --- a/src/cli/util.h +++ b/src/cli/util.h @@ -1,10 +1,13 @@ #pragma once #include +#include #include #include +#include +#include #include namespace uenv { @@ -23,6 +26,10 @@ struct squashfs_image { util::expected validate_squashfs_image(const std::string& path); +// Create registry based on configuration +util::expected +create_registry_from_config(const configuration& config); + } // namespace uenv template <> class fmt::formatter { diff --git a/src/site/site.cpp b/src/site/site.cpp index 728b400b..a75977b2 100644 --- a/src/site/site.cpp +++ b/src/site/site.cpp @@ -49,7 +49,7 @@ std::string default_namespace() { return "deploy"; } -util::expected +static util::expected registry_listing(const std::string& nspace) { using json = nlohmann::json; @@ -112,7 +112,7 @@ registry_listing(const std::string& nspace) { return store; } -std::string registry_url() { +static std::string registry_url() { return "jfrog.svc.cscs.ch/uenv"; } @@ -163,4 +163,34 @@ get_credentials(std::optional username, return oras::credentials{.username = uname.value(), .token = token_string}; } +// cscs-specific registry implementation +struct cscs_registry { + util::expected + listing(const std::string& nspace) const { + return registry_listing(nspace); + } + + std::string url() const { + return registry_url(); + } + + bool supports_search() const { + return true; + } + + uenv::registry_type type() const { + return uenv::registry_type::site; + } + + util::expected + manifest(const std::string&, const uenv::uenv_label&) const { + // TODO: fill this out + return {}; + } +}; + +uenv::registry create_site_registry() { + return uenv::registry{cscs_registry{}}; +} + } // namespace site diff --git a/src/site/site.h b/src/site/site.h index 0778d022..6ea9e237 100644 --- a/src/site/site.h +++ b/src/site/site.h @@ -1,9 +1,11 @@ #pragma once +#include #include #include #include +#include #include #include #include @@ -20,13 +22,11 @@ std::optional get_username(); // default namespace for image deployment std::string default_namespace(); -util::expected -registry_listing(const std::string& nspace); - -std::string registry_url(); - util::expected, std::string> get_credentials(std::optional username, std::optional token); +// Create site-specific registry (JFrog implementation) +uenv::registry create_site_registry(); + } // namespace site diff --git a/src/uenv/oras.cpp b/src/uenv/oras.cpp index 850e42a2..ee162af0 100644 --- a/src/uenv/oras.cpp +++ b/src/uenv/oras.cpp @@ -4,6 +4,9 @@ #include #include +#include +#include + #include #include #include @@ -11,6 +14,8 @@ #include #include +#include +#include #include #include #include @@ -20,6 +25,19 @@ namespace uenv { namespace oras { +namespace impl { + +void add_credentials(std::vector& args, + const std::optional token) { + if (token) { + args.push_back("--password"); + args.push_back(token->token); + args.push_back("--username"); + args.push_back(token->username); + } +} +} // namespace impl + using opt_creds = std::optional; // stores the "result" of calling the oras cli tool as an external process. @@ -148,6 +166,25 @@ run_oras_async(std::vector args, return util::run(args, runpath); } +// Wraps `oras discover` +// Oras discover returns a list of manifests that refer to a tag: +// oras discover --format=json --artifact-type=uenv/meta +// localhost:5862/deploy/cluster/zen3/app/1.0:v1 +// { +// "manifests": [ +// { +// ... +// "digest": "sha256:....", +// "artifactType": "uenv/meta", +// ... +// } +// ] +// } +// +// NOTE: we currently restrict the search to 'uenv/meta' artifacts +// +// returns a list of shas in the form "sha256:<64 characters>" +// The list should be of length 0 or 1 util::expected, error> discover(const std::string& registry, const std::string& nspace, const uenv_record& uenv, const opt_creds token) { @@ -157,12 +194,8 @@ discover(const std::string& registry, const std::string& nspace, std::vector args = {"discover", "--format", "json", "--artifact-type", "uenv/meta", address}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); + auto result = run_oras(args); if (result.returncode) { @@ -198,12 +231,8 @@ pull_digest(const std::string& registry, const std::string& nspace, std::vector args{"pull", "--output", destination.string(), address}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); + auto proc = run_oras(args); if (proc.returncode) { @@ -214,6 +243,29 @@ pull_digest(const std::string& registry, const std::string& nspace, return {}; } +util::expected pull_sha(const std::string& registry, + const std::string& nspace, + const uenv_record& uenv, + const opt_creds token) { + auto address = + fmt::format("{}/{}/{}/{}/{}/{}:{}", registry, nspace, uenv.system, + uenv.uarch, uenv.name, uenv.version, uenv.tag); + + spdlog::debug("oras::pull_sha: {}", address); + + std::vector args{"resolve", address}; + impl::add_credentials(args, token); + + auto proc = run_oras(args); + + if (proc.returncode) { + spdlog::error("unable to resolve image sha with oras: {}", proc.stderr); + return util::unexpected{create_error(proc)}; + } + + return uenv::sha256(uenv::parse_oras_sha256(proc.stdout).value()); +} + util::expected pull_tag(const std::string& registry, const std::string& nspace, const uenv_record& uenv, @@ -230,12 +282,8 @@ util::expected pull_tag(const std::string& registry, spdlog::debug("oras::pull_tag: {}", address); std::vector args{"pull", "--concurrency", "10", "--output", destination.string(), address}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); + auto proc = run_oras_async(args); if (!proc) { @@ -305,13 +353,7 @@ util::expected push_tag(const std::string& registry, // Prepare the arguments for the push command std::vector args{"push", "--concurrency", "10"}; - - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); // Add artifact type and annotations args.push_back("--artifact-type"); @@ -399,12 +441,7 @@ util::expected push_meta(const std::string& registry, // Prepare the arguments for the attach command std::vector args{"attach"}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } + impl::add_credentials(args, token); // Add artifact type and annotations for metadata args.push_back("--artifact-type"); @@ -491,5 +528,37 @@ copy(const std::string& registry, const std::string& src_nspace, return {}; } +util::expected +manifest(const std::string& registry, const uenv_record& uenv, + const std::optional token) { + auto address = fmt::format("{}/{}/{}/{}/{}:{}", registry, uenv.system, + uenv.uarch, uenv.name, uenv.version, uenv.tag); + std::vector args = {"manifest", "fetch", address}; + impl::add_credentials(args, token); + + auto result = run_oras(args); + + if (result.returncode) { + spdlog::error("oras manifest fetch returncode={} stderr='{}'", + result.returncode, result.stderr); + return util::unexpected{create_error(result)}; + } + + std::vector manifests; + using json = nlohmann::json; + uenv::manifest M; + try { + const auto raw = json::parse(result.stdout); + const auto& layer = raw["layers"][0]; + M.squashfs_digest = parse_oras_sha256(layer["digest"]).value(); + M.squashfs_bytes = layer["size"]; + } catch (std::exception& e) { + spdlog::error("error parsing oras manifest fetch output: {}", e.what()); + return util::unexpected(generic_error(e.what())); + } + + return M; +} + } // namespace oras } // namespace uenv diff --git a/src/uenv/oras.h b/src/uenv/oras.h index 2eb5d717..9243e624 100644 --- a/src/uenv/oras.h +++ b/src/uenv/oras.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -60,12 +61,22 @@ push_meta(const std::string& registry, const std::string& nspace, const uenv_label& label, const std::filesystem::path& meta_path, const std::optional token = std::nullopt); +util::expected +pull_sha(const std::string& registry, const std::string& nspace, + const uenv_record& uenv, + const std::optional token = std::nullopt); + util::expected copy(const std::string& registry, const std::string& src_nspace, const uenv_record& src_uenv, const std::string& dst_nspace, const uenv_record& dst_uenv, const std::optional token = std::nullopt); +// combine registry +util::expected +manifest(const std::string& registry, const uenv_record& uenv, + const std::optional token = std::nullopt); + } // namespace oras } // namespace uenv diff --git a/src/uenv/parse.cpp b/src/uenv/parse.cpp index 24e5bf8c..fe5fc538 100644 --- a/src/uenv/parse.cpp +++ b/src/uenv/parse.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace uenv { @@ -607,4 +608,69 @@ parse_config_line(const std::string& arg) { return result; } +util::expected +parse_registry_url(const std::string& arg) { + // TODO: parse + // leading [https://] + // name [/name]+ + // where name = [alpha,integer,_,-,.,~]+, see: + // https://blog.sucuri.net/2023/01/bad-paths-the-importance-of-using-valid-url-characters.html + // + // for initial testing, assume that the url is correct + return arg; +} + +// tokens that can appear in names +// names are used for uenv names, versions, tags +bool is_sha_tok(lex::tok t) { + return t == lex::tok::symbol || t == lex::tok::integer; +}; + +// parse the sha from a string of the form: +// sha256:34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15 +util::expected +parse_oras_sha256(const std::string& arg) { + const auto line = util::strip(arg); + + auto L = lex::lexer(line); + + if (L.peek().spelling != "sha") { + return util::unexpected{parse_error{ + L.string(), "oras sha is must start with 'sha256", L.peek()}}; + } + L.next(); + if (L.peek().spelling != "256") { + return util::unexpected{parse_error{ + L.string(), "oras sha is must start with 'sha256", L.peek()}}; + } + L.next(); + if (L.peek().kind != lex::tok::colon) { + return util::unexpected{parse_error{ + L.string(), + fmt::format("unexpected symbol '{}'", L.peek().spelling), + L.peek()}}; + } + L.next(); + + auto result = parse_string(L, "sha256", is_sha_tok); + if (!result) { + return result; + } + + const auto r = *result; + if (L.peek().kind != lex::tok::end) { + return util::unexpected{parse_error{ + L.string(), + fmt::format("unexpected symbol '{}'", L.peek().spelling), + L.peek()}}; + } + if (r.size() != 64) { + return util::unexpected{parse_error{ + L.string(), fmt::format("sha is not 64 characters in length"), + L.peek()}}; + } + + return result; +} + } // namespace uenv diff --git a/src/uenv/parse.h b/src/uenv/parse.h index 2b7a4a62..950653ab 100644 --- a/src/uenv/parse.h +++ b/src/uenv/parse.h @@ -67,4 +67,10 @@ parse_registry_entry(const std::string& in); util::expected parse_config_line(const std::string& arg); +util::expected +parse_registry_url(const std::string& arg); + +util::expected +parse_oras_sha256(const std::string& arg); + } // namespace uenv diff --git a/src/uenv/registry.cpp b/src/uenv/registry.cpp new file mode 100644 index 00000000..869525cb --- /dev/null +++ b/src/uenv/registry.cpp @@ -0,0 +1,76 @@ +#include "registry.h" + +#include +#include + +#include + +namespace uenv { + +// Generic OCI Registry implementation +struct oci_registry { + private: + std::string url_; + + public: + explicit oci_registry(const std::string& url) : url_(url) { + } + + util::expected + listing(const std::string& nspace) const { + spdlog::debug("OCI registry does not support listing for namespace: {}", + nspace); + // Return empty repository - OCI registries generally don't support + // search + return uenv::create_repository(); + } + + std::string url() const { + return url_; + } + + bool supports_search() const { + return false; + } + + registry_type type() const { + return registry_type::oci; + } + + util::expected + manifest(const std::string&, const uenv::uenv_label&) const { + // TODO: fill this out + return {}; + } +}; + +// Factory function implementation +registry create_registry(const std::string& url, registry_type type) { + spdlog::debug("creating registry {}::{}", type, url); + switch (type) { + case registry_type::oci: + return registry{oci_registry{url}}; + case registry_type::site: + // Site-specific implementation will be created in site module + spdlog::error("Site registry type should be created via site module"); + // Return a default OCI registry as fallback + return registry{oci_registry{url}}; + } + // Should never reach here, but provide a fallback + return registry{oci_registry{url}}; +} + +// Parse registry type from string +util::expected +parse_registry_type(const std::string& type) { + if (type == "oci") { + return registry_type::oci; + } else if (type == "site") { + return registry_type::site; + } else { + return util::unexpected(fmt::format( + "Invalid registry type: {}. Valid types are: oci, site", type)); + } +} + +} // namespace uenv diff --git a/src/uenv/registry.h b/src/uenv/registry.h new file mode 100644 index 00000000..e4154543 --- /dev/null +++ b/src/uenv/registry.h @@ -0,0 +1,167 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include + +namespace uenv { + +struct manifest { + sha256 digest; + sha256 squashfs_digest; + size_t squashfs_bytes; + std::optional meta_digest; + std::string respository; + std::string tag; +}; + +enum class registry_type { + oci, // Generic OCI registry (Docker Hub, etc.) + site // Site-specific implementation (JFrog, etc.) +}; + +// Concept for registry implementations +// Any type T that implements these operations can be used as a registry +template +concept RegistryImpl = requires(T registry, const std::string& nspace, + const uenv::uenv_label& label) { + { + registry.listing(nspace) + } -> std::convertible_to>; + { registry.url() } -> std::convertible_to; + { registry.supports_search() } -> std::convertible_to; + { registry.type() } -> std::convertible_to; + { + registry.manifest(nspace, label) + } -> std::convertible_to>; +}; + +// Type-erased registry class using value semantics +class registry { + public: + template + registry(T impl) : impl_(std::make_unique>(std::move(impl))) { + } + + registry(registry&& other) = default; + + registry(const registry& other) : impl_(other.impl_->clone()) { + } + + registry& operator=(registry&& other) = default; + registry& operator=(const registry& other) { + return *this = registry(other); + } + + // Get listing of images in a namespace + // Returns empty repository for registries that don't support search + util::expected + listing(const std::string& nspace) const { + return impl_->listing(nspace); + } + + std::string url() const { + return impl_->url(); + } + + bool supports_search() const { + return impl_->supports_search(); + } + + registry_type type() const { + return impl_->type(); + } + + util::expected + manifest(const std::string& nspace, const uenv_label& label) const { + return impl_->manifest(nspace, label); + } + + private: + struct interface { + virtual ~interface() = default; + virtual std::unique_ptr clone() = 0; + virtual util::expected + listing(const std::string& nspace) const = 0; + virtual std::string url() const = 0; + virtual bool supports_search() const = 0; + virtual registry_type type() const = 0; + virtual util::expected + manifest(const std::string& nspace, const uenv_label& label) const = 0; + }; + + std::unique_ptr impl_; + + template struct wrap : interface { + explicit wrap(const T& impl) : wrapped(impl) { + } + explicit wrap(T&& impl) : wrapped(std::move(impl)) { + } + + virtual std::unique_ptr clone() override { + return std::make_unique>(wrapped); + } + + virtual util::expected + listing(const std::string& nspace) const override { + return wrapped.listing(nspace); + } + + virtual std::string url() const override { + return wrapped.url(); + } + + virtual bool supports_search() const override { + return wrapped.supports_search(); + } + + virtual registry_type type() const override { + return wrapped.type(); + } + + virtual util::expected + manifest(const std::string& nspace, + const uenv_label& label) const override { + return wrapped.manifest(nspace, label); + } + + T wrapped; + }; +}; + +// Factory function to create registry implementations +registry create_registry(const std::string& url, registry_type type); + +// Parse registry type from string +util::expected +parse_registry_type(const std::string& type_str); + +} // namespace uenv + +template <> class fmt::formatter { + public: + // parse format specification and store it: + constexpr auto parse(format_parse_context& ctx) { + return ctx.end(); + } + // format a value using stored specification: + template + constexpr auto format(uenv::registry_type const& type, + FmtContext& ctx) const { + using uenv::registry_type; + + switch (type) { + case registry_type::oci: + return fmt::format_to(ctx.out(), "oci"); + case registry_type::site: + return fmt::format_to(ctx.out(), "site"); + } + return fmt::format_to(ctx.out(), "unknown"); + } +}; diff --git a/src/uenv/settings.cpp b/src/uenv/settings.cpp index ec209c0e..5102c305 100644 --- a/src/uenv/settings.cpp +++ b/src/uenv/settings.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,8 @@ const std::string config_file_default = # set the path to the local uenv repository #repo = /users/bobsmith/uenv +#registry = ... + # by default uenv will choose whether to use color based on your environment. #color=true #color=false @@ -35,6 +38,12 @@ config_base merge(const config_base& lhs, const config_base& rhs) { .repo = lhs.repo ? lhs.repo : rhs.repo ? rhs.repo : std::nullopt, + .registry = lhs.registry ? lhs.registry + : rhs.registry ? rhs.registry + : std::nullopt, + .registry_type = lhs.registry_type ? lhs.registry_type + : rhs.registry_type ? rhs.registry_type + : std::nullopt, .color = lhs.color ? lhs.color : rhs.color ? rhs.color : std::nullopt, @@ -44,6 +53,8 @@ config_base merge(const config_base& lhs, const config_base& rhs) { config_base default_config(const envvars::state& env) { return { .repo = default_repo_path(env), + .registry = "jfrog.svc.cscs.ch/uenv", // Default site registry URL + .registry_type = "site", .color = color::default_color(env), }; } @@ -53,7 +64,8 @@ generate_configuration(const config_base& base) { configuration config; // set the repo path - // initialise to unset + + // initialise to unset, and then set if the path provided by base is valid. config.repo = {}; if (base.repo) { if (auto path = parse_path(base.repo.value())) { @@ -68,6 +80,16 @@ generate_configuration(const config_base& base) { } } + config.registry = base.registry; + + // set registry type + auto registry_type_result = + parse_registry_type(base.registry_type.value_or("site")); + if (!registry_type_result) { + return util::unexpected(registry_type_result.error()); + } + config.registry_type_val = *registry_type_result; + // toggle color output config.color = base.color.value_or(false); @@ -82,44 +104,57 @@ read_config_file(const std::filesystem::path& path, } util::expected -load_user_config(const envvars::state& calling_env) { +load_user_config(const envvars::state& calling_env, + const std::optional& config_path) { namespace fs = std::filesystem; - auto home_env = calling_env.get("HOME"); - auto xdg_env = calling_env.get("XDG_CONFIG_HOME"); - // return an empty config if no configuration path can be determined - if (!home_env && !xdg_env) { - spdlog::warn("unable to find default configuration location, neither " - "HOME nor XDG_CONFIG_HOME are defined."); - return config_base{}; - } - const auto config_path = - xdg_env ? (fs::path(xdg_env.value()) / "uenv") - : (fs::path(home_env.value()) / ".config/uenv"); - const auto config_file = config_path / "config"; - - auto create_config_file = [](const auto& path) { - auto fid = std::ofstream(path); - fid << config_file_default << std::endl; - }; - if (!fs::exists(config_path)) { - spdlog::info("creating configuration path {}", config_path); - std::error_code ec; - fs::create_directories(config_path, ec); - if (ec) { - spdlog::error("unable to create config path: {}", ec.message()); + fs::path config_file; + + // if a custom config path is provided, use it directly + if (config_path) { + spdlog::info("using custom configuration file {}", + config_path->string()); + + if (!(fs::exists(*config_path) && fs::is_regular_file(*config_path))) { + return util::unexpected(fmt::format( + "custom configuration file '{}' is not a regular file", + config_path->string())); + } + + config_file = config_path.value(); + } else { + auto home_env = calling_env.get("HOME"); + auto xdg_env = calling_env.get("XDG_CONFIG_HOME"); + // return an empty config if no configuration path can be determined + if (!home_env && !xdg_env) { + spdlog::warn( + "unable to find default configuration location, neither " + "HOME nor XDG_CONFIG_HOME are defined."); + return config_base{}; + } + const auto config_dir = + xdg_env ? (fs::path(xdg_env.value()) / "uenv") + : (fs::path(home_env.value()) / ".config/uenv"); + config_file = config_dir / "config"; + + if (!fs::exists(config_dir)) { + spdlog::info("creating configuration path {}", config_dir); + std::error_code ec; + fs::create_directories(config_dir, ec); + if (ec) { + spdlog::error("unable to create config path: {}", ec.message()); + return config_base{}; + } + } + if (!fs::exists(config_file)) { + spdlog::info("creating configuration file {}", config_file); + auto fid = std::ofstream(config_file); + fid << config_file_default << std::endl; return config_base{}; } - spdlog::info("creating configuration file {}", config_file); - create_config_file(config_file); - return config_base{}; - } else if (!fs::exists(config_file)) { - spdlog::info("creating configuration file {}", config_file); - create_config_file(config_file); - return config_base{}; } - spdlog::info("opening configuration file {}", config_file); + spdlog::info("parsing configuration file {}", config_file); auto result = impl::read_config_file(config_file, calling_env); if (!result) { @@ -127,7 +162,7 @@ load_user_config(const envvars::state& calling_env) { "error opening '{}': {}", config_file.string(), result.error())}; } - return *result; + return result.value(); } namespace impl { @@ -172,6 +207,8 @@ read_config_file(const std::filesystem::path& path, fid.close(); + spdlog::debug("finished parsing file"); + // build a config from the key value config_base config; for (auto [key, value] : settings) { @@ -186,9 +223,22 @@ read_config_file(const std::filesystem::path& path, } else { return util::unexpected( fmt::format("invalid configuration value '{}={}': color " - "muste be true or false", + "must be true or false", key, value)); } + } else if (key == "registry") { + // an empty value is interpretted as unsetting the registry + if (value == "") { + config.registry = std::nullopt; + } else if (const auto p = parse_registry_url(value)) { + config.registry = p.value(); + } else { + return util::unexpected( + fmt::format("invalid reguistry url '{}={}': {}", key, value, + p.error().message())); + } + } else if (key == "registry-type") { + config.registry_type = value; } else { return util::unexpected( fmt::format("invalid configuration parameter '{}'", key)); diff --git a/src/uenv/settings.h b/src/uenv/settings.h index 3cefb9dc..0b67fcf7 100644 --- a/src/uenv/settings.h +++ b/src/uenv/settings.h @@ -4,6 +4,9 @@ #include #include +#include + +#include #include #include @@ -11,6 +14,8 @@ namespace uenv { struct config_base { std::optional repo; + std::optional registry; + std::optional registry_type; std::optional color; }; @@ -26,8 +31,10 @@ struct config_line { // read configuration from the user configuration file // the location of the config file is determined using XDG_CONFIG_HOME or HOME -util::expected -load_user_config(const envvars::state&); +// if config_path is provided, use it directly instead of the default locations +util::expected load_user_config( + const envvars::state&, + const std::optional& config_path = std::nullopt); // get the default configuration config_base default_config(const envvars::state& calling_env); @@ -36,6 +43,8 @@ config_base merge(const config_base& lhs, const config_base& rhs); struct configuration { std::optional repo; + std::optional registry; + registry_type registry_type_val; bool color; configuration& operator=(const configuration&) = default; }; @@ -44,3 +53,19 @@ util::expected generate_configuration(const config_base& base); } // namespace uenv + +template <> class fmt::formatter { + public: + // parse format specification and store it: + constexpr auto parse(format_parse_context& ctx) { + return ctx.end(); + } + // format a value using stored specification: + template + constexpr auto format(uenv::config_base const& cfg, FmtContext& ctx) const { + return fmt::format_to( + ctx.out(), "[repo: {}, registry: {}, registry_type: {}, color: {}]", + cfg.repo.value_or("()"), cfg.registry.value_or("()"), + cfg.registry_type.value_or("()"), cfg.color.value_or("()")); + } +}; diff --git a/test/integration/cli.bats b/test/integration/cli.bats index 64574bfe..4d0ffc00 100644 --- a/test/integration/cli.bats +++ b/test/integration/cli.bats @@ -27,6 +27,8 @@ function setup() { function teardown() { : + # this will tear down the test registry if it was started inside the test + #teardown_test_registry } @test "noargs" { @@ -467,3 +469,72 @@ EOF [ ! -d $UENV_REPO_PATH/images/$sha ] } +@test "docker registry setup teardown" { + # Skip if Docker is not available + if ! registry_is_available; then + skip "Docker not available for registry tests" + fi + + # Set up test registry + setup_test_registry + + # Verify registry is accessible + run curl -s "http://localhost:$REGISTRY_PORT/v2/" + assert_success + + # Verify environment variables are set + [ -n "$REGISTRY_URL" ] + [ -n "$REGISTRY_PORT" ] + [ -n "$REGISTRY_CONTAINER" ] + + # Store container name before teardown + local container_name="$REGISTRY_CONTAINER" + + # Test teardown + teardown_test_registry + + # Verify container is removed + run docker ps -a --filter "name=$container_name" --format "{{.Names}}" + refute_output --partial "$container_name" +} + +@test "oci registry push pull" { + # Set up test registry + setup_test_registry + + # Skip if Docker is not available + if ! registry_is_available; then + skip "Docker not available for registry tests" + fi + + local work=$(mktemp -d $TMP/push-test-XXXXXX) + + # Create a test repository for the source image + local repo=$work/repo + run uenv repo create $repo + assert_success + + # create a config file that points to the working repo and registry + local config=$work/config + echo "repo = $repo" > $config + echo "registry = $REGISTRY_URL" >> $config + echo "registry-type = oci" >> $config + + # Push the image to the oci registry + sqfs_path=$SQFS_LIB/apptool/standalone/app42.squashfs + run uenv -vv --config=$config image push $sqfs_path deploy::app/1.0:v1@cluster%zen3 + assert_success + assert_output --partial "successfully pushed" + + # Pull the image from the oci registry + run uenv -vvv --config=$config image pull deploy::app/1.0:v1@cluster%zen3 + assert_success + + # Verify the image was pulled successfully + run uenv --config=$config image ls test-app --no-header + assert_success + assert_output --partial "test-app/1.0:v1" + assert_output --partial "zen3" + assert_output --partial "test" +} + diff --git a/test/integration/common.bash b/test/integration/common.bash index 9dd466b7..5db1d575 100755 --- a/test/integration/common.bash +++ b/test/integration/common.bash @@ -33,3 +33,57 @@ function run_sbatch() { run_sbatch_unchecked "$@" [ "${status}" -eq 0 ] } + +# Docker-based test registry setup and teardown functions +function setup_test_registry() { + # Check if Docker is available + if ! command -v docker &> /dev/null; then + skip "Docker not available for registry tests" + fi + + # Use a random high port to avoid conflicts + export REGISTRY_PORT=$((5000 + RANDOM % 1000)) + export REGISTRY_URL="localhost:$REGISTRY_PORT" + export REGISTRY_CONTAINER="uenv-test-registry-$$" + + log "Setting up test registry on port $REGISTRY_PORT" + + # Start Zot registry container + docker run -d \ + --name "$REGISTRY_CONTAINER" \ + -p "$REGISTRY_PORT:5000" \ + ghcr.io/project-zot/zot:latest || { + skip "Failed to start Docker registry container" + } + + # Wait for registry to be ready (up to 30 seconds) + local count=0 + while [ $count -lt 30 ]; do + if curl -s "http://localhost:$REGISTRY_PORT/v2/" > /dev/null 2>&1; then + log "Test registry is ready at $REGISTRY_URL" + return 0 + fi + sleep 1 + count=$((count + 1)) + done + + # If we get here, the registry didn't start properly + docker logs "$REGISTRY_CONTAINER" || true + teardown_test_registry + skip "Test registry failed to become ready" +} + +function teardown_test_registry() { + if [ -n "${REGISTRY_CONTAINER:-}" ]; then + log "Cleaning up test registry container: $REGISTRY_CONTAINER" + docker stop "$REGISTRY_CONTAINER" >/dev/null 2>&1 || true + docker rm "$REGISTRY_CONTAINER" >/dev/null 2>&1 || true + unset REGISTRY_CONTAINER + unset REGISTRY_PORT + unset REGISTRY_URL + fi +} + +function registry_is_available() { + command -v docker &> /dev/null +} diff --git a/test/unit/parse.cpp b/test/unit/parse.cpp index b34481e8..4706aecc 100644 --- a/test/unit/parse.cpp +++ b/test/unit/parse.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -362,11 +361,7 @@ TEST_CASE("parse registry entry", "[parse]") { "build/eiger/zen2/prgenv-gnu/24.11/1529952520", }) { auto r = uenv::parse_registry_entry(s); - if (!r) { - fmt::println("{}", r.error().message()); - } else { - REQUIRE(r); - } + REQUIRE(r); } } @@ -473,3 +468,45 @@ TEST_CASE("config_line", "[parse]") { REQUIRE(!result); } } + +TEST_CASE("oras_sha", "[parse]") { + { + auto sha = uenv::parse_oras_sha256( + "sha256:" + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15"); + REQUIRE(sha); + REQUIRE( + sha.value() == + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15"); + } + { + auto sha = uenv::parse_oras_sha256( + "sha256:" + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c1"); + REQUIRE(!sha); + } + { + auto sha = uenv::parse_oras_sha256( + "sha255:" + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15"); + REQUIRE(!sha); + } + { + auto sha = uenv::parse_oras_sha256( + "sha256@" + "34c77667fa06e4c73bf98e357a8823b7eb0a2a38a84b22d03fed5b45387f9c15"); + REQUIRE(!sha); + } + { + auto sha = uenv::parse_oras_sha256("sha256:" + "34c77667fa06e4c73bf98e357a8823b7eb0" + "a2a38a84b22d03fed5b45387f9c15-"); + REQUIRE(!sha); + } + { + auto sha = uenv::parse_oras_sha256("sha256:" + "34c77667fa06e4c73bf98e357a8823b7eb0" + "a2a-38a84b22d03fed5b45387f9c15"); + REQUIRE(!sha); + } +}