Merge pull request #358 from xylar/add-mache-deploy-workflow-tests

Add mache deploy workflow tests

Author: xylar

.github/workflows/build_workflow.yml

@@ -109,3 +109,43 @@ jobs:
         name: Build Sphinx Docs
         run: |
           pixi run -e ${{ matrix.pixi-env }} bash -e -c "cd docs && DOCS_VERSION=test make versioned-html"
+
+  workflow-tests:
+    name: workflow tests - python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        include:
+          - python-version: "3.10"
+            pixi-env: py310
+          - python-version: "3.14"
+            pixi-env: py314
+      fail-fast: false
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@master
+        with:
+          cancel_others: ${{ env.CANCEL_OTHERS }}
+          paths_ignore: ${{ env.PATHS_IGNORE }}
+
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        uses: actions/checkout@v6
+
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Set up Pixi
+        uses: prefix-dev/setup-pixi@v0.9.4
+        with:
+          pixi-version: v0.62.2
+          cache: ${{ hashFiles('pixi.lock') != '' }}
+          environments: ${{ matrix.pixi-env }}
+
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Install mache
+        run: |
+          pixi run -e ${{ matrix.pixi-env }} python -m pip install --no-deps --no-build-isolation -vv -e .
+
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Run workflow tests
+        run: |
+          pixi run -e ${{ matrix.pixi-env }} pytest workflow_tests

conda/recipe/recipe.yaml

@@ -2,7 +2,7 @@ schema_version: 1
 
 context:
   name: mache
-  version: 3.0.0
+  version: 3.0.1
 
 package:
   name: ${{ name|lower }}
@@ -42,6 +42,10 @@ tests:
       run:
         - python ${{ python_min }}.*
     script:
+      - mache deploy init --help
+      - mache deploy update --help
+      - mache deploy run --help
+      - mache jigsaw install --help
       - mache sync diags --help
 
 about:

mache/__main__.py

@@ -40,3 +40,7 @@ def _build_parser() -> argparse.ArgumentParser:
     add_deploy_subparser(subparsers)
     add_jigsaw_subparser(subparsers)
     return parser
+
+
+if __name__ == '__main__':
+    main()

mache/deploy/bootstrap.py

@@ -2,12 +2,29 @@
 
 import argparse
 import os
+import platform
 import shlex
 import shutil
 import subprocess
 import sys
 from pathlib import Path
 
+CONDA_PLATFORM_MAP = {
+    ('linux', 'x86_64'): 'linux-64',
+    ('linux', 'aarch64'): 'linux-aarch64',
+    ('linux', 'ppc64le'): 'linux-ppc64le',
+    ('osx', 'x86_64'): 'osx-64',
+    ('osx', 'arm64'): 'osx-arm64',
+}
+
+LOCAL_MACHE_SOURCE_ENV = 'MACHE_LOCAL_SOURCE_PATH'
+PIXI_ENV_VARS_TO_UNSET = (
+    'PIXI_PROJECT_MANIFEST',
+    'PIXI_PROJECT_ROOT',
+    'PIXI_ENVIRONMENT_NAME',
+    'PIXI_IN_SHELL',
+)
+
 
 def check_call(
     commands,
@@ -197,7 +214,7 @@ def build_pixi_shell_hook_prefix(*, pixi_exe: str, pixi_toml: str) -> str:
     running a nested shell process.
     """
     hook_cmd = (
-        'env -u PIXI_PROJECT_MANIFEST -u PIXI_PROJECT_ROOT '
+        f'{build_pixi_env_unset_prefix()} '
         f'{shlex.quote(pixi_exe)} shell-hook -s bash -m '
         f'{shlex.quote(pixi_toml)}'
     )
@@ -353,7 +370,7 @@ def _run(log_filename):
 
         cmd_install = (
             f'cd "{bootstrap_dir}" && '
-            'env -u PIXI_PROJECT_MANIFEST -u PIXI_PROJECT_ROOT '
+            f'{build_pixi_env_unset_prefix()} '
             f'"{pixi_exe}" install'
         )
         check_call(cmd_install, log_filename, quiet)
@@ -380,7 +397,7 @@ def _run(log_filename):
 
         cmd_install = (
             f'cd "{bootstrap_dir}" && '
-            'env -u PIXI_PROJECT_MANIFEST -u PIXI_PROJECT_ROOT '
+            f'{build_pixi_env_unset_prefix()} '
             f'"{pixi_exe}" install'
         )
         check_call(cmd_install, log_filename, quiet)
@@ -569,6 +586,23 @@ def _default_pixi_path():
     return os.path.join(home, '.pixi', 'bin', 'pixi')
 
 
+def build_pixi_env_unset_prefix():
+    return 'env ' + ' '.join(f'-u {name}' for name in PIXI_ENV_VARS_TO_UNSET)
+
+
+def _get_pixi_platform():
+    system = platform.system().lower()
+    if system == 'darwin':
+        system = 'osx'
+    machine = platform.machine().lower()
+    try:
+        return CONDA_PLATFORM_MAP[(system, machine)]
+    except KeyError as exc:
+        raise ValueError(
+            f'Unsupported platform for pixi bootstrap: {system} {machine}'
+        ) from exc
+
+
 def _install_pixi(*, log_filename, quiet, pixi_bin_dir=None):
     env_prefix_parts = [
         # Avoid modifying shell rc files during bootstrap.
@@ -672,6 +706,7 @@ def _write_bootstrap_pixi_toml_with_mache(
         '[workspace]',
         f'name = "{name}"',
         'channels = ["conda-forge"]',
+        f'platforms = ["{_get_pixi_platform()}"]',
         'channel-priority = "strict"',
         '',
         '[dependencies]',
@@ -712,6 +747,32 @@ def _clone_mache_repo(
 
     build_root.mkdir(parents=True, exist_ok=True)
 
+    local_source = os.environ.get(LOCAL_MACHE_SOURCE_ENV)
+    if local_source:
+        source_repo = Path(
+            os.path.abspath(os.path.expanduser(local_source))
+        ).resolve()
+        if not source_repo.is_dir():
+            raise RuntimeError(
+                f'Local mache source override does not exist: {source_repo}'
+            )
+
+        try:
+            os.symlink(source_repo, repo_dir, target_is_directory=True)
+        except OSError:
+            shutil.copytree(
+                source_repo,
+                repo_dir,
+                ignore=shutil.ignore_patterns(
+                    '.git',
+                    '.pixi',
+                    'deploy_tmp',
+                    '__pycache__',
+                    '*.pyc',
+                ),
+            )
+        return
+
     commands = (
         f'cd "{build_root!s}" && '
         + "GIT_SSH_COMMAND='ssh -oBatchMode=yes' git clone "

mache/deploy/run.py

@@ -15,6 +15,7 @@
 from mache.jigsaw import deploy_jigsawpy
 
 from .bootstrap import (
+    build_pixi_env_unset_prefix,
     build_pixi_shell_hook_prefix,
     check_call,
     check_location,
@@ -986,7 +987,7 @@ def _pixi_install(
     # - Otherwise pixi uses its own defaults (typically under $HOME).
     cmd = (
         f'cd {shlex.quote(project_dir)} && '
-        'env -u PIXI_PROJECT_MANIFEST -u PIXI_PROJECT_ROOT '
+        f'{build_pixi_env_unset_prefix()} '
         f'{shlex.quote(pixi_exe)} install'
     )
     check_call(cmd, log_filename=log_filename, quiet=quiet)

mache/deploy/templates/deploy.py.j2

@@ -300,6 +300,10 @@ def _bootstrap_url(
     mache_fork=None,
     mache_branch=None,
 ):
+    override_url = str(os.environ.get('MACHE_BOOTSTRAP_URL', '')).strip()
+    if override_url:
+        return override_url
+
     if mache_fork is not None and mache_branch is not None:
         # Raw file from a fork/branch
         return f'https://raw.githubusercontent.com/{mache_fork}/{mache_branch}/{BOOTSTRAP_RELPATH}'  # noqa: E501
@@ -456,6 +460,7 @@ def _run_mache_deploy_run(pixi_exe, repo_root, mache_run_argv):
 
     cmd = (
         f'env -u PIXI_PROJECT_MANIFEST -u PIXI_PROJECT_ROOT '
+        f'-u PIXI_ENVIRONMENT_NAME -u PIXI_IN_SHELL '
         f'{shlex.quote(pixi_exe)} run -m {shlex.quote(pixi_toml)} bash -lc '
         f'{shlex.quote("cd " + repo_root + " && " + mache_cmd)}'
     )

mache/deploy/templates/load.sh.j2

@@ -53,6 +53,7 @@ if [[ -n "${MACHE_DEPLOY_RUNTIME_VERSION_CMD:-}" ]]; then
   # Use a non-login shell so pixi-provided PATH/env are not reset by site profiles.
   probe_output="$(
     env -u PIXI_PROJECT_MANIFEST -u PIXI_PROJECT_ROOT \
+      -u PIXI_ENVIRONMENT_NAME -u PIXI_IN_SHELL \
       "${PIXI}" run -m "${PIXI_TOML}" -- \
       bash -c "${MACHE_DEPLOY_RUNTIME_VERSION_CMD}" 2>&1
   )"
@@ -90,6 +91,7 @@ fi
 echo "loading pixi env..."
 # Use `shell-hook` (not `pixi run`) so activation applies to this shell.
 eval "$(env -u PIXI_PROJECT_MANIFEST -u PIXI_PROJECT_ROOT \
+  -u PIXI_ENVIRONMENT_NAME -u PIXI_IN_SHELL \
   "${PIXI}" shell-hook -s bash -m "${PIXI_TOML}")"
 echo "   pixi env loaded."
 

mache/version.py

@@ -1,2 +1,2 @@
-__version_info__ = (3, 0, 0)
+__version_info__ = (3, 0, 1)
 __version__ = '.'.join(str(vi) for vi in __version_info__)

tests/test_deploy_bootstrap.py

@@ -61,3 +61,48 @@ def fake_popen(commands, **kwargs):
     assert recorded['commands'] == 'printf hello'
     assert 'text' not in recorded['kwargs']
     assert recorded['kwargs']['universal_newlines'] is False
+
+
+def test_write_bootstrap_pixi_toml_with_mache_includes_platform(
+    monkeypatch, tmp_path: Path
+):
+    monkeypatch.setattr(bootstrap, '_get_pixi_platform', lambda: 'linux-64')
+
+    pixi_toml = tmp_path / 'pixi.toml'
+    bootstrap._write_bootstrap_pixi_toml_with_mache(
+        pixi_toml_path=pixi_toml,
+        software='polaris',
+        mache_version='3.0.0',
+        python_version='3.12',
+    )
+
+    text = pixi_toml.read_text(encoding='utf-8')
+    assert 'platforms = ["linux-64"]' in text
+    assert 'mache = "==3.0.0"' in text
+
+
+def test_clone_mache_repo_uses_local_source_override(
+    monkeypatch, tmp_path: Path
+):
+    monkeypatch.chdir(tmp_path)
+
+    source_repo = tmp_path / 'local-mache'
+    source_repo.mkdir()
+    (source_repo / 'pixi.toml').write_text(
+        '[workspace]\nname = "mache-dev"\n',
+        encoding='utf-8',
+    )
+
+    monkeypatch.setenv(bootstrap.LOCAL_MACHE_SOURCE_ENV, str(source_repo))
+
+    bootstrap._clone_mache_repo(
+        mache_fork='ignored',
+        mache_branch='ignored',
+        log_filename=str(tmp_path / 'bootstrap.log'),
+        quiet=True,
+        recreate=False,
+    )
+
+    cloned_repo = tmp_path / 'deploy_tmp' / 'build_mache' / 'mache'
+    assert cloned_repo.exists()
+    assert (cloned_repo / 'pixi.toml').is_file()

workflow_tests/fixtures/deploy_overrides/config.yaml.j2

@@ -0,0 +1,21 @@
+project:
+  software: "toyflow"
+  version: "0.1.0"
+  machine: null
+  runtime_version_cmd: >-
+    python -c 'from toyflow.version import __version__; print(__version__)'
+
+pixi:
+  deploy: true
+  prefix: "pixi-env"
+  channels:
+    - conda-forge
+  mpi: "nompi"
+  install_dev_software: true
+
+spack:
+  deploy: false
+  supported: false
+
+jigsaw:
+  enabled: false