update v5 for new pipelines (eth-cscs#228)

bcumming · web-flow · commit 6c277d2f031a · 2025-06-10T17:39:51.000+02:00
diff --git a/.flake8 b/.flake8
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -1,25 +1,16 @@
 name: lint
 
-on:
-  pull_request:
-    branches: [master, main]
+on: [push, pull_request]
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Install Tools
+    - name: Install uv
       run: |
-        python -m pip install --upgrade pip
-        python -m pip install black flake8 mypy
-    - name: Black
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+    - name: ruff
       run: |
-        black --check --verbose stackinator unittests
-    - name: flake8
-      run: |
-        flake8 --count --show-source --statistics .
+        uvx ruff format --check
+        uvx ruff check
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -3,20 +3,13 @@ name: Stackinator CI
 on: [push, pull_request]
 
 jobs:
-  unittestpy36:
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ['3.6']
+  unittestpy:
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Bootstrap
+    - name: Install uv
       run: |
-        ./bootstrap.sh
+        curl -LsSf https://astral.sh/uv/install.sh | sh
     - name: Generic Unittests
       run: |
         ./test_stackinator.py
diff --git a/bin/stack-config b/bin/stack-config
@@ -1,12 +1,18 @@
-#!/usr/bin/env python3
-
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#   "jinja2",
+#   "jsonschema",
+#   "pyYAML",
+# ]
+# ///
 
 import pathlib
 import sys
 
 prefix = pathlib.Path(__file__).parent.parent.resolve()
-external = prefix / 'external'
-sys.path = [prefix.as_posix(), external.as_posix()] + sys.path
+sys.path = [prefix.as_posix()] + sys.path
 
 from stackinator.main import main
 
diff --git a/bootstrap.sh b/bootstrap.sh
diff --git a/docs/readme.md b/docs/readme.md
@@ -1,15 +1,11 @@
 The documentation for Stackinator is built from the markdown files in this path using MkDocs and MkDocs-material.
 You can view the latest documentation online at [github.io](https://eth-cscs.github.io/stackinator/)
 
-To build a copy locally, first install `mkdocs-material`, e.g.:
+To view work in progress docs, run the serve script and follow the link it provides to view a local copy of the docs in your browser.
 ```bash
-python3 -m venv docs-env
-source docs-env/bin/activate
-pip install mkdocs-material
+./serve
 ```
 
-Then in the root of this project, build the docs and view them with your favourite browser:
-```bash
-mkdocs build
-firefox site/index.html
-```
+> [!IMPORTANT]
+> to run the serve script, you need to first install [uv](https://docs.astral.sh/uv/getting-started/installation/).
+
diff --git a/docs/recipes.md b/docs/recipes.md
@@ -25,13 +25,32 @@ spack:
     commit: releases/v0.20
 modules: true
 description: "HPC development tools for building MPI applications with the GNU compiler toolchain"
+version: 2
 ```
 
 * `name`: a plain text name for the environment
 * `store`: the location where the environment will be mounted.
 * `spack`: which spack repository to use for installation.
 * `modules`: _optional_ enable/diasble module file generation (default `true`).
 * `description`: _optional_ a string that describes the environment (default empty).
+* `version`:  _default = 1_ the version of the uenv recipe (see below)
+
+!!! note "uenv recipe versions"
+    Stackinator 6 introduces breaking changes to the uenv recipe format, introduced to support Spack v1.0.
+
+    We have started versioning uenv recipes:
+
+    * **version 1**: original uenv recipes for Spack v0.23 and earlier, supported by Stackinator version 5.
+    * **version 2**: uenv recipes for Spack v1.0 and later, supported by Stackinator version 6.
+
+    The default version is 1, so that old recipes that do not set a version are supported.
+
+    !!! warning "You must set version 2 explicitly to use Spack v1.0"
+
+    !!! warning "Version 1 recipes must be configured using Stackinator v5"
+        Version 5 of Stackinator is maintained in the `releases/v5` branch of stackinator.
+
+        You must also use the `releases/v5` branch of [Alps cluster config](https://github.com/eth-cscs/alps-cluster-config).
 
 ## Compilers
 
diff --git a/lint b/lint
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+uvx ruff format
+uvx ruff check --fix
diff --git a/pyproject.toml b/pyproject.toml
@@ -2,13 +2,11 @@
 requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta"
 
-[tool.black]
-line-length = 120
-
-[tool.isort]
-profile = "black"
-skip = ["external/"]
-
 [tool.ruff]
 line-length = 120
 extend-exclude = ["external/", "unittests/recipes"]
+exclude = ["unittests/recipes/with-repo/repo/packages"]
+
+[tool.ruff.lint]
+select = ["E", "F"]
+ignore = ["E203"]
diff --git a/stackinator/builder.py b/stackinator/builder.py
@@ -183,7 +183,7 @@ def generate(self, recipe):
 
         # Clone the spack repository if it has not already been checked out
         if not (spack_path / ".git").is_dir():
-            self._logger.info(f'spack: clone repository {spack["repo"]}')
+            self._logger.info(f"spack: clone repository {spack['repo']}")
 
             # clone the repository
             capture = subprocess.run(
@@ -195,12 +195,12 @@ def generate(self, recipe):
             self._logger.debug(capture.stdout.decode("utf-8"))
 
             if capture.returncode != 0:
-                self._logger.error(f'error cloning the repository {spack["repo"]}')
+                self._logger.error(f"error cloning the repository {spack['repo']}")
                 capture.check_returncode()
 
         # Fetch the specific branch
         if spack["commit"]:
-            self._logger.info(f'spack: fetch branch/commit {spack["commit"]}')
+            self._logger.info(f"spack: fetch branch/commit {spack['commit']}")
             capture = subprocess.run(
                 ["git", "-C", spack_path, "fetch", "origin", spack["commit"]],
                 shell=False,
@@ -210,12 +210,12 @@ def generate(self, recipe):
             self._logger.debug(capture.stdout.decode("utf-8"))
 
             if capture.returncode != 0:
-                self._logger.debug(f'unable to change to the fetch {spack["commit"]}')
+                self._logger.debug(f"unable to change to the fetch {spack['commit']}")
                 capture.check_returncode()
 
         # Check out a branch or commit if one was specified
         if spack["commit"]:
-            self._logger.info(f'spack: checkout branch/commit {spack["commit"]}')
+            self._logger.info(f"spack: checkout branch/commit {spack['commit']}")
             capture = subprocess.run(
                 ["git", "-C", spack_path, "checkout", spack["commit"]],
                 shell=False,
@@ -225,7 +225,7 @@ def generate(self, recipe):
             self._logger.debug(capture.stdout.decode("utf-8"))
 
             if capture.returncode != 0:
-                self._logger.debug(f'unable to change to the requested commit {spack["commit"]}')
+                self._logger.debug(f"unable to change to the requested commit {spack['commit']}")
                 capture.check_returncode()
 
         # get the spack commit
diff --git a/stackinator/etc/add-compiler-links.py b/stackinator/etc/add-compiler-links.py
@@ -69,7 +69,7 @@ def has_prefix(path, prefix):
 for c in compilers:
     local_paths = set([os.path.dirname(v) for k, v in c["paths"].items() if v is not None])
     paths += local_paths
-    print(f'adding compiler {c["spec"]} -> {[p for p in local_paths]}')
+    print(f"adding compiler {c['spec']} -> {[p for p in local_paths]}")
 
 # find unique paths and concatenate them
 pathstring = ":".join(set(paths))
diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py
@@ -164,7 +164,7 @@ def __repr__(self):
         return f'envvars.ListEnvVar("{self.name}", {self._updates})'
 
     def __str__(self):
-        return f"(\"{self.name}\": [{','.join([str(u) for u in self._updates])}])"
+        return f'("{self.name}": [{",".join([str(u) for u in self._updates])}])'
 
 
 class ScalarEnvVar(EnvVar):
@@ -458,11 +458,11 @@ def view_impl(args):
 
         for c in compilers:
             source_paths = list(set([os.path.abspath(v) for _, v in c["paths"].items() if v is not None]))
-            target_paths = [os.path.join(os.path.join(root_path, 'bin'), os.path.basename(f)) for f in source_paths]
+            target_paths = [os.path.join(os.path.join(root_path, "bin"), os.path.basename(f)) for f in source_paths]
             for src, dst in zip(source_paths, target_paths):
-                print(f'creating compiler symlink: {src} -> {dst}')
+                print(f"creating compiler symlink: {src} -> {dst}")
                 if os.path.exists(dst):
-                    print(f'  first removing {dst}')
+                    print(f"  first removing {dst}")
                     os.remove(dst)
                 os.symlink(src, dst)
 
diff --git a/stackinator/main.py b/stackinator/main.py
@@ -14,7 +14,7 @@
 
 def generate_logfile_name(name=""):
     idstr = f"{time.localtime()}{os.getpid}{platform.uname()}"
-    return f'log{name}_{hashlib.md5(idstr.encode("utf-8")).hexdigest()}'
+    return f"log{name}_{hashlib.md5(idstr.encode('utf-8')).hexdigest()}"
 
 
 def configure_logging(logfile):
@@ -45,7 +45,7 @@ def log_header(args):
 
 
 def make_argparser():
-    parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from " "a recipe."))
+    parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe."))
     parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}")
     parser.add_argument("-b", "--build", required=True, type=str)
     parser.add_argument("--no-bwrap", action="store_true", required=False)
@@ -76,7 +76,7 @@ def main():
 
         builder.generate(recipe)
 
-        root_logger.info("\nConfiguration finished, run the following to build the " "environment:\n")
+        root_logger.info("\nConfiguration finished, run the following to build the environment:\n")
         root_logger.info(f"cd {builder.path}")
         root_logger.info(
             'env --ignore-environment http_proxy="$http_proxy" https_proxy="$https_proxy" no_proxy="$no_proxy"'
diff --git a/stackinator/recipe.py b/stackinator/recipe.py
@@ -51,7 +51,29 @@ def __init__(self, args):
         # required config.yaml file
         self.config = self.path / "config.yaml"
 
-        # set the recipe-defined mount point
+        # check the version of the recipe
+        if self.config["version"] != 1:
+            rversion = self.config["version"]
+            if rversion == 2:
+                self._logger.error(
+                    "\nThe recipe is an new version 2 recipe for Spack v1.0 and later.\n"
+                    "This version of Stackinator supports Spack v0.23 and earlier.\n"
+                    "Use the main branch of stackinator:\n"
+                    "    git switch main\n\n"
+                    "If this recipe is to be used with Spack v0.23, then please add the field 'version: 1' to\n"
+                    "config.yaml in your recipe.\n\n"
+                    "For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration\n"
+                )
+                raise RuntimeError("incompatible uenv recipe version")
+            else:
+                self._logger.error(
+                    f"\nThe config.yaml file sets an unknown recipe version={rversion}.\n"
+                    "This version of Stackinator supports version 2 recipes.\n\n"
+                    "For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration\n"
+                )
+                raise RuntimeError("incompatible uenv recipe version")
+
+        # override the mount point if defined as a CLI argument
         if args.mount:
             self.config["store"] = args.mount
 
diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json
@@ -56,6 +56,10 @@
                 {"type" : "null"}
             ],
             "default": null
+        },
+        "version" : {
+            "type": "number",
+            "default": 1
         }
     }
 }
diff --git a/test_stackinator.py b/test_stackinator.py
@@ -1,11 +1,19 @@
-#!/usr/bin/env python3
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#   "jinja2",
+#   "jsonschema",
+#   "pytest",
+#   "pyYAML",
+# ]
+# ///
 
 import pathlib
 import sys
 
 prefix = pathlib.Path(__file__).parent.resolve()
-external = prefix / "external"
-sys.path = [prefix.as_posix(), external.as_posix()] + sys.path
+sys.path = [prefix.as_posix()] + sys.path
 
 import pytest  # noqa: E402
 
