Add --no-spack flag and clean up spack error messages

Author: xylar

docs/design/mache_deploy.md

@@ -195,13 +195,16 @@ maintainers to do full shared-environment deployments when needed.
     “software” spack env)
 - `deploy.py` exposes CLI flags such as:
   - `--deploy-spack` (force-enable spack deployment)
+  - `--no-spack` (disable all spack use for a single run)
   - `--spack-path <path>` (temporary override for `spack.spack_path`)
 
 **Precedence**
 
-1. If the user passes `--deploy-spack`, spack deployment is enabled regardless
-   of the config default.
-2. Otherwise, fall back to the `config.yaml.j2` default.
+1. If the user passes `--no-spack`, all spack use is disabled for that run.
+2. Otherwise, if the user passes `--deploy-spack`, spack deployment is enabled
+   regardless of the config default.
+3. Otherwise, fall back to the `config.yaml.j2` default for deployment and
+   reuse any supported pre-existing spack environments in load scripts.
 
 For spack checkout path resolution:
 

docs/users_guide/deploy.md

@@ -174,7 +174,8 @@ Important settings:
 - `pixi.prefix`: required in practice.
 - `pixi.channels`: required and must be non-empty.
 - `spack.spack_path`: required when Spack support is enabled and no hook or
-  CLI override provides it.
+  CLI override provides it, unless the user disables Spack for that run with
+  `--no-spack`.
 - `jigsaw.enabled`: optional.
 - `hooks`: optional and disabled unless explicitly configured.
 
@@ -492,13 +493,16 @@ Fix:
 
 Cause:
 
-- Spack support was enabled but no path was provided in config, hooks, or the
-  CLI.
+- Spack support was enabled for this run, but no path was provided in config,
+  hooks, or the CLI.
 
 Fix:
 
 - Set `spack.spack_path`, write it in a hook through
   `ctx.runtime["spack"]["spack_path"]`, or pass `--spack-path`.
+- If this should be a Pixi-only run, pass `--no-spack`.
+- A `pre_spack` hook may also disable Spack for one run by returning
+  `{"spack": {"supported": false, "software": {"supported": false}}}`.
 
 ### `project.version` is still `dynamic`
 
@@ -521,4 +525,4 @@ Cause:
 
 Fix:
 
-- Use `source load_<software>*.sh` from `bash`.
+- Use `source load_<software>*.sh` from `bash`.

mache/deploy/run.py

@@ -29,8 +29,11 @@
 from .spack import (
     deploy_spack_envs,
     deploy_spack_software_env,
+    get_effective_spack_config,
     load_existing_spack_envs,
     load_existing_spack_software_env,
+    spack_disabled_for_run,
+    spack_should_deploy_for_run,
 )
 
 
@@ -263,15 +266,20 @@ def run_deploy(args: argparse.Namespace) -> None:
     # Future wiring: spack stages (no-ops unless implemented/configured)
     hook_registry.run_hook('pre_spack', ctx)
 
-    spack_cfg = config.get('spack', {})
-    if not isinstance(spack_cfg, dict):
-        spack_cfg = {}
+    if getattr(args, 'deploy_spack', False) and getattr(
+        args, 'no_spack', False
+    ):
+        raise ValueError(
+            'Cannot combine --deploy-spack and --no-spack in the same run.'
+        )
 
-    deploy_spack = bool(spack_cfg.get('deploy')) or bool(
-        getattr(args, 'deploy_spack', False)
-    )
+    spack_cfg = get_effective_spack_config(ctx=ctx)
+    deploy_spack = spack_should_deploy_for_run(ctx=ctx, spack_cfg=spack_cfg)
 
-    if deploy_spack:
+    if spack_disabled_for_run(ctx=ctx):
+        spack_results = []
+        spack_software_env = None
+    elif deploy_spack:
         spack_results = deploy_spack_envs(
             ctx=ctx,
             toolchain_pairs=toolchain_pairs,

mache/deploy/spack.py

@@ -46,6 +46,57 @@ class SpackSoftwareEnvResult:
     path_setup: str
 
 
+def get_effective_spack_config(*, ctx: DeployContext) -> dict:
+    """Return deploy config merged with runtime Spack overrides."""
+
+    spack_cfg = ctx.config.get('spack', {})
+    if not isinstance(spack_cfg, dict):
+        spack_cfg = {}
+
+    merged = dict(spack_cfg)
+
+    software_cfg = merged.get('software', {})
+    if software_cfg is None:
+        software_cfg = {}
+    if isinstance(software_cfg, dict):
+        merged['software'] = dict(software_cfg)
+
+    runtime_spack = ctx.runtime.get('spack', {})
+    if not isinstance(runtime_spack, dict):
+        return merged
+
+    for key, value in runtime_spack.items():
+        if key == 'software' and isinstance(value, dict):
+            base = merged.get('software', {})
+            if not isinstance(base, dict):
+                base = {}
+            merged['software'] = {**base, **value}
+        else:
+            merged[key] = value
+
+    return merged
+
+
+def spack_disabled_for_run(*, ctx: DeployContext) -> bool:
+    """Return True when the CLI explicitly disables all Spack use."""
+
+    return bool(getattr(ctx.args, 'no_spack', False))
+
+
+def spack_should_deploy_for_run(
+    *, ctx: DeployContext, spack_cfg: dict
+) -> bool:
+    """Return True when Spack environments should be deployed for this run."""
+
+    if spack_disabled_for_run(ctx=ctx):
+        return False
+
+    if bool(getattr(ctx.args, 'deploy_spack', False)):
+        return True
+
+    return bool(spack_cfg.get('deploy'))
+
+
 def deploy_spack_software_env(
     *,
     ctx: DeployContext,
@@ -62,7 +113,7 @@ def deploy_spack_software_env(
     No CLI flags control this environment.
     """
 
-    spack_cfg = ctx.config.get('spack', {})
+    spack_cfg = get_effective_spack_config(ctx=ctx)
     if not isinstance(spack_cfg, dict):
         return None
 
@@ -72,9 +123,7 @@ def deploy_spack_software_env(
     if not isinstance(software_cfg, dict):
         raise ValueError('spack.software must be a mapping if provided')
 
-    deploy_spack = bool(spack_cfg.get('deploy')) or bool(
-        getattr(ctx.args, 'deploy_spack', False)
-    )
+    deploy_spack = spack_should_deploy_for_run(ctx=ctx, spack_cfg=spack_cfg)
 
     software_supported = bool(software_cfg.get('supported'))
 
@@ -210,7 +259,7 @@ def deploy_spack_envs(
         A list of SpackDeployResult, one per (compiler, mpi)
     """
 
-    spack_cfg = ctx.config.get('spack', {})
+    spack_cfg = get_effective_spack_config(ctx=ctx)
     if not isinstance(spack_cfg, dict):
         return []
 
@@ -220,9 +269,7 @@ def deploy_spack_envs(
     if not isinstance(software_cfg, dict):
         raise ValueError('spack.software must be a mapping if provided')
 
-    deploy_spack = bool(spack_cfg.get('deploy')) or bool(
-        getattr(ctx.args, 'deploy_spack', False)
-    )
+    deploy_spack = spack_should_deploy_for_run(ctx=ctx, spack_cfg=spack_cfg)
 
     library_supported = bool(spack_cfg.get('supported'))
 
@@ -364,7 +411,7 @@ def load_existing_spack_envs(
 ) -> list[SpackDeployResult]:
     """Load pre-existing Spack library environments for load scripts."""
 
-    spack_cfg = ctx.config.get('spack', {})
+    spack_cfg = get_effective_spack_config(ctx=ctx)
     if not isinstance(spack_cfg, dict):
         return []
 
@@ -374,15 +421,17 @@ def load_existing_spack_envs(
 
     if not toolchain_pairs:
         raise ValueError(
-            'Spack library environments are supported but no toolchain pairs '
-            'were resolved. Provide toolchain.compiler/toolchain.mpi or '
-            '--compiler/--mpi.'
+            'Spack library environments are enabled for this run but no '
+            'toolchain pairs were resolved. Provide toolchain.compiler/'
+            'toolchain.mpi or --compiler/--mpi, or disable Spack for this '
+            'run with --no-spack or a pre_spack hook.'
         )
 
     spack_path = _resolve_spack_path(
         ctx=ctx,
         spack_cfg=spack_cfg,
-        reason='support is enabled (load scripts require existing envs)',
+        reason='support is enabled for this run (load scripts will reuse '
+        'existing envs)',
     )
 
     e3sm_hdf5_netcdf = _get_machine_bool(
@@ -446,7 +495,7 @@ def load_existing_spack_software_env(
 ) -> SpackSoftwareEnvResult | None:
     """Load a pre-existing Spack software environment for load scripts."""
 
-    spack_cfg = ctx.config.get('spack', {})
+    spack_cfg = get_effective_spack_config(ctx=ctx)
     if not isinstance(spack_cfg, dict):
         return None
 
@@ -462,13 +511,17 @@ def load_existing_spack_software_env(
 
     if ctx.machine is None:
         raise ValueError(
-            'Spack software environment is supported but machine is not known.'
+            'Spack software environment is enabled for this run but machine '
+            'is not known. Pass --no-spack or disable '
+            'spack.software.supported '
+            'in a pre_spack hook for Pixi-only runs.'
         )
 
     spack_path = _resolve_spack_path(
         ctx=ctx,
         spack_cfg=spack_cfg,
-        reason='support is enabled (load scripts require existing envs)',
+        reason='support is enabled for this run (load scripts will reuse '
+        'existing envs)',
     )
 
     compiler, mpi = _resolve_software_toolchain(
@@ -568,7 +621,8 @@ def _resolve_spack_path(
             '--spack-path, '
             "ctx.runtime['spack']['spack_path']"
             ' in a hook (preferred) or set spack.spack_path in '
-            'deploy/config.yaml.j2'
+            'deploy/config.yaml.j2. To bypass Spack entirely for this run, '
+            'pass --no-spack.'
         )
     return os.path.abspath(os.path.expanduser(os.path.expandvars(spack_path)))
 

mache/deploy/templates/cli_spec.json.j2

@@ -44,6 +44,13 @@
       "help": "Deploy all supported Spack environments (overrides spack.deploy in deploy/config.yaml.j2).",
       "route": ["deploy", "run"]
     },
+    {
+      "flags": ["--no-spack"],
+      "dest": "no_spack",
+      "action": "store_true",
+      "help": "Disable all Spack use for this run, including reuse of existing Spack environments.",
+      "route": ["deploy", "run"]
+    },
     {
       "flags": ["--spack-path"],
       "dest": "spack_path",

mache/deploy/templates/config.yaml.j2.j2

@@ -112,6 +112,9 @@ spack:
   #
   # This can be forced on at runtime with the `mache deploy run` CLI flag:
   #   --deploy-spack
+  #
+  # Passing --no-spack disables all Spack use for a single run, including
+  # reuse of pre-existing environments for load scripts.
   deploy: false
 
   # Whether this target repository supports a Spack *library* environment.
@@ -142,12 +145,17 @@ spack:
   # In practice, most target repositories should set this dynamically in the
   # `pre_spack()` hook (e.g., based on machine config) by writing:
   #   ctx.runtime['spack']['spack_path'] = <path>
+  # Hooks may also disable Spack for a single run by returning:
+  #   ctx.runtime['spack']['supported'] = False
+  #   ctx.runtime['spack']['software']['supported'] = False
   # This config value is a fallback.
   #
   # Required (either via hook/runtime or here) when spack.deploy (or
   # --deploy-spack) is used and at least one supported spack environment is
-  # enabled.
+  # enabled, or when an existing Spack environment will be reused for load
+  # scripts.
   # Can also be overridden temporarily via: --spack-path <path>
+  # To bypass Spack entirely for one run, pass: --no-spack
   spack_path: null
 
   # Prefix for spack environment names.