modular sqfs

docs/interfaces.md

@@ -12,7 +12,6 @@ For example, the Spack configuration is in the following files when a stack has
 ```
 /user-environment
 ├─ config
-│  ├─ compilers.yaml
 │  ├─ repos.yaml
 │  ├─ packages.yaml
 │  └─ upstreams.yaml
@@ -31,7 +30,6 @@ Notes on the configuration files:
       system:
         install_tree: /user-environment
     ```
-* `compilers.yaml`: includes all compilers that were installed in the `gcc:` and `llvm:` sections of the `compilers.yaml` recipe file. Note that the `bootstrap` compiler is not included.
 * `packages.yaml`: refers to the external packages that were used to configure the recipe: both the defaults in the cluster configuration, and any additional packages that were set in the recipe.
 * `repos.yaml`: points to the custom Spack repository:
     ```yaml

stackinator/builder.py

@@ -266,19 +266,30 @@ def generate(self, recipe):
 
         make_user_template = jinja_env.get_template("Make.user")
         with (self.path / "Make.user").open("w") as f:
+            base_uenvs = [e["image"] for e in recipe.base_uenv["compilers"]]
+            if "gpu" in recipe.base_uenv:
+                base_uenvs += [recipe.base_uenv["gpu"]["image"]]
             f.write(
                 make_user_template.render(
                     spack_version=spack_version,
                     build_path=self.path,
                     store=recipe.mount,
                     no_bwrap=recipe.no_bwrap,
+                    base_uenv=base_uenvs,
                     verbose=False,
                 )
             )
             f.write("\n")
 
         etc_path = self.root / "etc"
-        for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py"]:
+        for f_etc in [
+            "Make.inc",
+            "bwrap-mutable-root.sh",
+            "bwrap-store.sh",
+            "envvars.py",
+            "gen_packages_yaml.py",
+            "squashfs-mount-wrapper.sh",
+        ]:
             shutil.copy2(etc_path / f_etc, self.path / f_etc)
 
         # used to configure both pre and post install hooks, if they are provided.
@@ -414,7 +425,7 @@ def generate(self, recipe):
 
         # Delete the store/repo path, if it already exists.
         # Do this so that incremental builds (though not officially supported) won't break if a repo is updated.
-        repo_dst = store_path / "repo"
+        repo_dst = store_path / "spack_repo/alps"
         self._logger.debug(f"creating the stack spack prepo in {repo_dst}")
         if repo_dst.exists():
             self._logger.debug(f"{repo_dst} exists ... deleting")
@@ -432,13 +443,14 @@ def generate(self, recipe):
                 """\
 repo:
   namespace: alps
+  api: v2.0
 """
             )
 
         # create the repository step 2: create the repos.yaml file in build_path/config
         repos_yaml_template = jinja_env.get_template("repos.yaml")
         with (config_path / "repos.yaml").open("w") as f:
-            repo_path = recipe.mount / "repo"
+            repo_path = recipe.mount / "spack_repo/alps"
             f.write(repos_yaml_template.render(repo_path=repo_path.as_posix(), verbose=False))
             f.write("\n")
 
@@ -457,19 +469,6 @@ def generate(self, recipe):
                     elif dst.exists():
                         self._logger.debug(f"  NOT installing package {pkg_path}")
 
-        # Generate the makefile and spack.yaml files that describe the compilers
-        compiler_files = recipe.compiler_files
-        compiler_path = self.path / "compilers"
-        compiler_path.mkdir(exist_ok=True)
-        with (compiler_path / "Makefile").open(mode="w") as f:
-            f.write(compiler_files["makefile"])
-
-        for name, yml in compiler_files["config"].items():
-            compiler_config_path = compiler_path / name
-            compiler_config_path.mkdir(exist_ok=True)
-            with (compiler_config_path / "spack.yaml").open(mode="w") as f:
-                f.write(yml)
-
         # generate the makefile and spack.yaml files that describe the environments
         environment_files = recipe.environment_files
         environments_path = self.path / "environments"
@@ -480,6 +479,7 @@ def generate(self, recipe):
         for name, yml in environment_files["config"].items():
             env_config_path = environments_path / name
             env_config_path.mkdir(exist_ok=True)
+            # packages.yaml is added in the makefile from the base uenv
             with (env_config_path / "spack.yaml").open(mode="w") as f:
                 f.write(yml)
 
@@ -490,14 +490,10 @@ def generate(self, recipe):
         generate_config_path.mkdir(exist_ok=True)
 
         # write generate-config/Makefile
-        all_compilers = [x for x in recipe.compilers.keys()]
-        release_compilers = [x for x in all_compilers if x != "bootstrap"]
         with (generate_config_path / "Makefile").open("w") as f:
             f.write(
                 make_config_template.render(
                     build_path=self.path.as_posix(),
-                    all_compilers=all_compilers,
-                    release_compilers=release_compilers,
                     verbose=False,
                 )
             )

stackinator/etc/Make.inc

@@ -16,7 +16,7 @@ store:
 	mkdir -p $(STORE)
 
 # Concretization
-%/spack.lock: %/spack.yaml %/compilers.yaml %/config.yaml %/packages.yaml
+%/spack.lock: %/spack.yaml %/config.yaml %/packages.yaml
 	$(SPACK_ENV) concretize -f
 
 # Generate Makefiles for the environment install

stackinator/etc/bwrap-mutable-root.sh

@@ -1,11 +1,21 @@
 #!/bin/bash
+
+set -euo pipefail
 args=()
 shopt -s dotglob
+
+# from /user-environment/foo/bar/baz store /user-environment as _top_level
+_top_level=$(echo $STORE | cut -d "/" -f 2 | xargs printf "/%s")
+
 for d in /*; do
+    # skip STORE
+    if [ "$d" = "${_top_level}" ]; then
+        continue
+    fi
     # skip invalid symlinks, as they will break bwrap
     if [ ! -L "$d" ] || [ -e "$d" ]; then
         args+=("--dev-bind" "$d" "$d")
     fi
 done
-PS1="\[\e[36;1m\]build-env >>> \[\e[0m\]" bwrap "${args[@]}" "$@"
 
+PS1="\[\e[36;1m\]build-env >>> \[\e[0m\]" bwrap "${args[@]}" "$@"

stackinator/etc/bwrap-store.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+mkdir -p $STORE
+
+bwrap --dev-bind / / \
+      --bind ${BUILD_ROOT}/store $STORE \
+      -- "$@"

stackinator/etc/gen_packages_yaml.py

@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+
+import argparse
+import json
+import subprocess
+import pathlib
+import yaml
+
+
+def compiler_extra_attributes(name, prefix):
+    """Find paths to compiler"""
+    if name == "gcc":
+        cc = "gcc"
+        cxx = "g++"
+        f90 = "gfortran"
+    elif name == "llvm":
+        cc = "clang"
+        cxx = "clang++"
+        f90 = None
+    elif name == "nvhpc":
+        cc = "nvc"
+        cxx = "nvc++"
+        f90 = "nvfortran"
+    else:
+        # this is not a compiler
+        return {}
+
+    def find(comp):
+        p = subprocess.run(
+            ["find", prefix, "-name", f"{comp}", "-path", "*/bin/*"],
+            shell=False,
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        return p.stdout.strip().decode("utf-8")
+
+    extra_attributes = {"extra_attributes": {"compilers": {"c": find(cc), "cxx": find(cxx)}}}
+    if f90 is not None:
+        extra_attributes["extra_attributes"]["compilers"]["fortran"] = find(f90)
+
+    return extra_attributes
+
+
+def gen_packages_impl(lock_file, env_path):
+    spack_lock = json.load(open(lock_file, "r"))
+
+    packages = {"packages": {}}
+
+    for dd in spack_lock["roots"]:
+        hash = dd["hash"]
+        # call subprocess to find install dir
+        spack_find_prefix = subprocess.run(
+            ["spack", "--color=never", "-e", env_path, "find", "--format={prefix}", f"/{hash}"],
+            shell=False,
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        spack_find_spec = subprocess.run(
+            ["spack", "--color=never", "-e", env_path, "find", "--format={name}|{version}|{variants}", f"/{hash}"],
+            shell=False,
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        name, version, variants = spack_find_spec.stdout.strip().decode("utf-8").split("|")
+        prefix = spack_find_prefix.stdout.strip().decode("utf-8")
+
+        packages["packages"][name] = {
+            "buildable": False,
+            "externals": [
+                {
+                    "spec": f"{name}@{version} {variants}",
+                    "prefix": prefix,
+                }
+            ],
+        }
+        # add `extra_attributes` for compilers
+        if name in ["gcc", "nvhpc", "llvm"]:
+            extra_attributes = compiler_extra_attributes(name, prefix)
+            packages["packages"][name]["externals"][0].update(extra_attributes)
+
+        return packages
+
+
+if __name__ == "__main__":
+    # parse CLI arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--lock-file", help="spack.lock", type=str)
+    parser.add_argument("--env-path", help="path to spack env", type=str)
+    parser.add_argument("--view", help="path to spack view", type=str)
+
+    args = parser.parse_args()
+
+    packages = gen_packages_impl(args.lock_file, args.env_path)
+
+    dst = pathlib.Path(args.view) / "packages.yaml"
+    with open(dst, "w") as f:
+        yaml.dump(packages, f)

stackinator/etc/squashfs-mount-wrapper.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+# Function to display usage
+usage() {
+	  echo "Usage: $0 --build-root=<build-root> --sqfs-images '<sqfs1>:<mnt1> <sqfs2>:<mnt2> ..' -- <command>"
+	  exit 1
+}
+
+# Initialize variables
+BUILD_ROOT=""
+SQFS_IMAGES=""
+
+# Parse options using getopt
+TEMP=$(getopt -o '' --long build-root: --long sqfs-images: -n "$0" -- "$@")
+if [ $? -ne 0 ]; then
+	  echo "Error parsing arguments" >&2
+	  usage
+fi
+
+# Reset the positional parameters to the short options
+eval set -- "$TEMP"
+
+# Extract options
+while true; do
+	  case "$1" in
+	      --build-root)
+		        BUILD_ROOT="$2"
+		        shift 2
+		        ;;
+	      --sqfs-images)
+		        SQFS_IMAGES="$2"
+		        shift 2
+		        ;;
+	      --)
+		        shift
+		        break
+		        ;;
+	      *)
+		        echo "Unknown option: $1"
+		        usage
+		        ;;
+	  esac
+done
+
+if [ -z "$BUILD_ROOT" ]; then
+	  echo "Error: --build-root is required" >&2
+	  usage
+fi
+
+if [ -z "$SQFS_IMAGES" ]; then
+    # no images to mount, skip squashfs-mount
+    exec "$@"
+fi
+
+read -ra array <<<"$SQFS_IMAGES"
+
+if [ ${#array[@]} -eq 0 ]; then
+    echo "no mountpoints specified, skip squashfs-mount"
+    exec "$@"
+fi
+
+build_root_mounts=""
+for elem in "${array[@]}"; do
+	  mount_point=${elem#*:}
+    sqfs=${elem%%:*}
+    tmp_mount_point="${BUILD_ROOT}/tmp/mounts/${mount_point}"
+	  mkdir -p ${tmp_mount_point}
+    build_root_mounts="${build_root_mounts} ${sqfs}:${tmp_mount_point}"
+done
+
+squashfs-mount $build_root_mounts -- "$@"