From abaee68a9cf9b50c00b6b8f440c82aecc0673580 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Wed, 4 Jun 2025 18:32:07 +0200 Subject: [PATCH 1/5] mount modular sqfs via json description --- meson.build | 1 + src/cli/run.cpp | 5 +++ src/cli/start.cpp | 4 +++ src/uenv/env.cpp | 26 ++++++++++++++-- src/uenv/modular_env.cpp | 66 ++++++++++++++++++++++++++++++++++++++++ src/uenv/modular_env.hpp | 21 +++++++++++++ src/uenv/uenv.h | 3 ++ 7 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 src/uenv/modular_env.cpp create mode 100644 src/uenv/modular_env.hpp diff --git a/meson.build b/meson.build index 1c48377c..dda21942 100644 --- a/meson.build +++ b/meson.build @@ -41,6 +41,7 @@ barkeep_dep = declare_dependency( lib_src = [ 'src/site/site.cpp', 'src/uenv/env.cpp', + 'src/uenv/modular_env.cpp', 'src/uenv/log.cpp', 'src/uenv/meta.cpp', 'src/uenv/oras.cpp', diff --git a/src/cli/run.cpp b/src/cli/run.cpp index 48fb8090..e1c0c729 100644 --- a/src/cli/run.cpp +++ b/src/cli/run.cpp @@ -67,6 +67,11 @@ You need to finish the current session by typing 'exit' or hitting ''.)" for (auto e : env->uenvs) { commands.push_back(fmt::format("{}:{}", e.second.sqfs_path.string(), e.second.mount_path)); + // sub-images + for (auto [sqfs, mnt] : e.second.sub_images) { + commands.push_back( + fmt::format("{}:{}", sqfs.string(), mnt.string())); + } } commands.push_back("--"); diff --git a/src/cli/start.cpp b/src/cli/start.cpp index 73787b9a..98cc6f2c 100644 --- a/src/cli/start.cpp +++ b/src/cli/start.cpp @@ -110,6 +110,10 @@ will not work, because it starts a new interactive shell.)", for (auto e : env->uenvs) { commands.push_back(fmt::format("{}:{}", e.second.sqfs_path.string(), e.second.mount_path)); + // sub-images + for (auto [sqfs, mnt] : e.second.sub_images) { + commands.push_back(fmt::format("{}:{}", sqfs.string(), mnt.string())); + } } // find the current shell (zsh, bash, etc) diff --git a/src/uenv/env.cpp b/src/uenv/env.cpp index f553c7f0..a2ffd717 100644 --- a/src/uenv/env.cpp +++ b/src/uenv/env.cpp @@ -23,6 +23,9 @@ #include #include +#include "modular_env.hpp" +#include "util/expected.h" + namespace uenv { using util::unexpected; @@ -116,12 +119,14 @@ concretise_env(const std::string& uenv_args, // and meta data (if they have meta data). std::unordered_map uenvs; + // std::vector> std::set used_mounts; std::set used_sqfs; for (auto& desc : *uenv_descriptions) { // determine the sqfs_path fs::path sqfs_path; - + // dependent images + std::vector> sub_images; // if a label was used to describe the uenv (e.g. "prgenv-gnu/24.7" // it has to be looked up in a repo. if (auto label = desc.label()) { @@ -169,7 +174,17 @@ concretise_env(const std::string& uenv_args, // otherwise an explicit filename was provided, e.g. // "/scratch/myimages/develp/store.squashfs" else { + // check if desc.filename is a json sqfs_path = fs::path(*desc.filename()); + if (sqfs_path.extension() == ".json") { + // NOTE: we ignore mount_path and read it from the sqfs file + auto menv = read_modular_env(sqfs_path, *repo_arg); + if (!menv) { + return unexpected(menv.error()); + } + sqfs_path = menv.value().sqfs_path; + sub_images = menv.value().sub_images; + } } sqfs_path = fs::absolute(sqfs_path); @@ -264,8 +279,13 @@ concretise_env(const std::string& uenv_args, used_sqfs.insert(sqfs_path); } - uenvs[name] = concrete_uenv{name, mount, sqfs_path, - meta.path, description, std::move(views)}; + uenvs[name] = concrete_uenv{.name = name, + .mount_path = mount, + .sqfs_path = sqfs_path, + .sub_images = sub_images, + .meta_path = meta.path, + .description = description, + .views = std::move(views)}; } // A dictionary with view name as a key, and a list of uenv that provide diff --git a/src/uenv/modular_env.cpp b/src/uenv/modular_env.cpp new file mode 100644 index 00000000..61c5af90 --- /dev/null +++ b/src/uenv/modular_env.cpp @@ -0,0 +1,66 @@ +#include "modular_env.hpp" +#include +#include +#include +#include +#include +#include + +namespace uenv { + +util::expected +read_modular_env(const std::filesystem::path& modular_uenv_json_path, + std::optional repo_arg) { + + using json = nlohmann::json; + std::ifstream f(modular_uenv_json_path.c_str()); + json data; + try { + data = json::parse(f); + } catch (std::exception& e) { + return util::unexpected( + fmt::format("error modular uenv json file {}: {}", + modular_uenv_json_path.string(), e.what())); + } + + if (!data.contains("root-image")) { + return util::unexpected( + fmt::format("error {} doesn't specify the root-image", + modular_uenv_json_path.string())); + } + + auto store = uenv::open_repository(*repo_arg); + if (!store) { + return util::unexpected( + fmt::format("unable to open repo: {}", store.error())); + } + + std::vector> + sub_images; + std::filesystem::path sqfs_path = + data["root-image"]["file"].get(); + std::filesystem::path mount_path = + data["root-image"]["prefix_path"].get(); + + // GPU image if present + if (data.contains("gpu")) { + std::filesystem::path image = data["gpu"]["file"]; + std::filesystem::path mount = data["gpu"]["prefix_path"]; + sub_images.push_back(std::make_tuple(image, mount)); + } + + if (data.contains("compilers")) { + for(auto entry: data["compilers"]) { + auto mount = entry["image"]["prefix_path"]; + auto sqfs = entry["image"]["file"]; + sub_images.push_back(std::make_tuple(sqfs,mount)); + } + } + + return modular_env_paths{ + .sqfs_path = sqfs_path, + .mount_path = mount_path, + .sub_images = sub_images}; +} + +} // namespace uenv diff --git a/src/uenv/modular_env.hpp b/src/uenv/modular_env.hpp new file mode 100644 index 00000000..cd4fc65d --- /dev/null +++ b/src/uenv/modular_env.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include +#include "uenv/uenv.h" + +namespace uenv { + + +struct modular_env_paths { + std::filesystem::path sqfs_path; + std::filesystem::path mount_path; + std::vector> sub_images; +}; + +util::expected +read_modular_env(const std::filesystem::path&, + std::optional); + +} // namespace uenv diff --git a/src/uenv/uenv.h b/src/uenv/uenv.h index 5996c35b..0d12434e 100644 --- a/src/uenv/uenv.h +++ b/src/uenv/uenv.h @@ -147,6 +147,9 @@ struct concrete_uenv { std::filesystem::path mount_path; /// the path of the squashfs image to be mounted std::filesystem::path sqfs_path; + /// subordinate images tuple(sqfs path, mount point) + std::vector> + sub_images; /// the path of the meta data - not set if no meta data path was found std::optional meta_path; From c77dd7d0f13025f8348c5a77b1b642253056a404 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Thu, 5 Jun 2025 15:47:49 +0200 Subject: [PATCH 2/5] change in modular uenv json desc --- src/uenv/modular_env.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uenv/modular_env.cpp b/src/uenv/modular_env.cpp index 61c5af90..e171c6fa 100644 --- a/src/uenv/modular_env.cpp +++ b/src/uenv/modular_env.cpp @@ -38,9 +38,9 @@ read_modular_env(const std::filesystem::path& modular_uenv_json_path, std::vector> sub_images; std::filesystem::path sqfs_path = - data["root-image"]["file"].get(); + data["root-image"]["image"]["file"].get(); std::filesystem::path mount_path = - data["root-image"]["prefix_path"].get(); + data["root-image"]["image"]["prefix_path"].get(); // GPU image if present if (data.contains("gpu")) { From 1ac583c0a2dbc2cc7e61214aee136b019ba05558 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Thu, 5 Jun 2025 16:00:42 +0200 Subject: [PATCH 3/5] root-image -> root --- src/uenv/modular_env.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/uenv/modular_env.cpp b/src/uenv/modular_env.cpp index e171c6fa..257e52bf 100644 --- a/src/uenv/modular_env.cpp +++ b/src/uenv/modular_env.cpp @@ -23,7 +23,7 @@ read_modular_env(const std::filesystem::path& modular_uenv_json_path, modular_uenv_json_path.string(), e.what())); } - if (!data.contains("root-image")) { + if (!data.contains("root")) { return util::unexpected( fmt::format("error {} doesn't specify the root-image", modular_uenv_json_path.string())); @@ -38,9 +38,9 @@ read_modular_env(const std::filesystem::path& modular_uenv_json_path, std::vector> sub_images; std::filesystem::path sqfs_path = - data["root-image"]["image"]["file"].get(); + data["root"]["image"]["file"].get(); std::filesystem::path mount_path = - data["root-image"]["image"]["prefix_path"].get(); + data["root"]["image"]["prefix_path"].get(); // GPU image if present if (data.contains("gpu")) { From da0b08cac77b6cb11d1e1c7d787262be44d156b3 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Thu, 5 Jun 2025 16:03:11 +0200 Subject: [PATCH 4/5] fix path --- src/uenv/modular_env.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uenv/modular_env.cpp b/src/uenv/modular_env.cpp index 257e52bf..093d6d98 100644 --- a/src/uenv/modular_env.cpp +++ b/src/uenv/modular_env.cpp @@ -44,8 +44,8 @@ read_modular_env(const std::filesystem::path& modular_uenv_json_path, // GPU image if present if (data.contains("gpu")) { - std::filesystem::path image = data["gpu"]["file"]; - std::filesystem::path mount = data["gpu"]["prefix_path"]; + std::filesystem::path image = data["gpu"]["image"]["file"]; + std::filesystem::path mount = data["gpu"]["image"]["prefix_path"]; sub_images.push_back(std::make_tuple(image, mount)); } From 0a2cfa2dc27ebf1d0994b46db73eb7cd136e2a44 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Thu, 5 Jun 2025 18:37:30 +0200 Subject: [PATCH 5/5] clang format --- src/cli/start.cpp | 3 ++- src/uenv/env.cpp | 5 +++-- src/uenv/modular_env.cpp | 23 +++++++++++------------ src/uenv/modular_env.hpp | 12 ++++++------ 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/cli/start.cpp b/src/cli/start.cpp index 98cc6f2c..14c806f6 100644 --- a/src/cli/start.cpp +++ b/src/cli/start.cpp @@ -112,7 +112,8 @@ will not work, because it starts a new interactive shell.)", e.second.mount_path)); // sub-images for (auto [sqfs, mnt] : e.second.sub_images) { - commands.push_back(fmt::format("{}:{}", sqfs.string(), mnt.string())); + commands.push_back( + fmt::format("{}:{}", sqfs.string(), mnt.string())); } } diff --git a/src/uenv/env.cpp b/src/uenv/env.cpp index a2ffd717..38dde6dc 100644 --- a/src/uenv/env.cpp +++ b/src/uenv/env.cpp @@ -126,7 +126,8 @@ concretise_env(const std::string& uenv_args, // determine the sqfs_path fs::path sqfs_path; // dependent images - std::vector> sub_images; + std::vector> + sub_images; // if a label was used to describe the uenv (e.g. "prgenv-gnu/24.7" // it has to be looked up in a repo. if (auto label = desc.label()) { @@ -180,7 +181,7 @@ concretise_env(const std::string& uenv_args, // NOTE: we ignore mount_path and read it from the sqfs file auto menv = read_modular_env(sqfs_path, *repo_arg); if (!menv) { - return unexpected(menv.error()); + return unexpected(menv.error()); } sqfs_path = menv.value().sqfs_path; sub_images = menv.value().sub_images; diff --git a/src/uenv/modular_env.cpp b/src/uenv/modular_env.cpp index 093d6d98..e97c31ef 100644 --- a/src/uenv/modular_env.cpp +++ b/src/uenv/modular_env.cpp @@ -1,16 +1,16 @@ #include "modular_env.hpp" -#include -#include #include #include #include +#include #include +#include namespace uenv { util::expected read_modular_env(const std::filesystem::path& modular_uenv_json_path, - std::optional repo_arg) { + std::optional repo_arg) { using json = nlohmann::json; std::ifstream f(modular_uenv_json_path.c_str()); @@ -50,17 +50,16 @@ read_modular_env(const std::filesystem::path& modular_uenv_json_path, } if (data.contains("compilers")) { - for(auto entry: data["compilers"]) { - auto mount = entry["image"]["prefix_path"]; - auto sqfs = entry["image"]["file"]; - sub_images.push_back(std::make_tuple(sqfs,mount)); - } + for (auto entry : data["compilers"]) { + auto mount = entry["image"]["prefix_path"]; + auto sqfs = entry["image"]["file"]; + sub_images.push_back(std::make_tuple(sqfs, mount)); + } } - return modular_env_paths{ - .sqfs_path = sqfs_path, - .mount_path = mount_path, - .sub_images = sub_images}; + return modular_env_paths{.sqfs_path = sqfs_path, + .mount_path = mount_path, + .sub_images = sub_images}; } } // namespace uenv diff --git a/src/uenv/modular_env.hpp b/src/uenv/modular_env.hpp index cd4fc65d..ad8ad45f 100644 --- a/src/uenv/modular_env.hpp +++ b/src/uenv/modular_env.hpp @@ -1,17 +1,17 @@ #pragma once -#include +#include "uenv/uenv.h" #include +#include #include -#include "uenv/uenv.h" namespace uenv { - struct modular_env_paths { - std::filesystem::path sqfs_path; - std::filesystem::path mount_path; - std::vector> sub_images; + std::filesystem::path sqfs_path; + std::filesystem::path mount_path; + std::vector> + sub_images; }; util::expected