simplifying test job structure, avoiding duplicate test runs between jobs

Author: tclose

.github/workflows/ci-cd.yml

@@ -55,6 +55,36 @@ jobs:
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:
+      - name: Set env for singularity
+        run: echo "RELEASE_VERSION=v3.7.1" >> $GITHUB_ENV
+      - name: Disable etelemetry
+        run:  echo "NO_ET=TRUE" >> $GITHUB_ENV
+      - name: Setup Singularity
+        uses: actions/checkout@v4
+        with:
+          repository: hpcng/singularity
+          ref: 'v3.7.1'
+          path: 'singularity'
+      - name: Setup GO
+        uses: actions/setup-go@v5
+        with:
+          go-version: '^1.13'
+      - name: Install OS deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.1 \
+          libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y
+      - name: Build
+        run: |
+          cd singularity
+          ./mconfig --without-suid -p /usr/local/
+          make -C ./builddir
+          sudo make -C ./builddir install
+          cd ..
+      - name: Echo singularity version
+        run: |
+          echo ${{ github.ref }}
+          singularity --version
       - name: Fetch repository
         uses: actions/checkout@v4
       - name: Fetch tags
@@ -66,14 +96,12 @@ jobs:
       - name: Update pip
         run: python -m pip install --upgrade pip
       - name: Install Pydra
-        run: pip install .[test]
+        run: pip install .[test,psij,dask]
       - name: Print version
         run: python -c "import pydra.engine; print(pydra.engine.__version__)"
-      - name: Disable etelemetry
-        run:  echo "NO_ET=TRUE" >> $GITHUB_ENV
       - name: Pytest
         run: |
-          pytest -vs -n auto pydra --doctest-modules --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
+          pytest -vs -n auto pydra --doctest-modules --with-psij --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
       - name: Upload to codecov
         run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW
 
@@ -101,148 +129,15 @@ jobs:
           pip install -e ".[test,dask]"
       - name: Run tests for Dask
         run: |
-          pytest -v --dask pydra/engine --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
-      - name: Upload to codecov
-        run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW
-
-  test-psijlocal:
-    needs: ['build']
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: ['3.11']
-      fail-fast: false
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository }}
-      - name: Fetch tags
-        run: git fetch --prune --unshallow
-      - name: Setup Python version ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies for PSI/J
-        run: |
-          pip install -e ".[test, psij]"
-      - name: Run tests for PSI/J
-        run: |
-          pytest --color=yes -vs --psij=local -n auto pydra/engine --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
-      - name: Upload to codecov
-        run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW
-
-  test-psijslurm:
-    needs: ['build']
-    strategy:
-      matrix:
-        python-version: [3.11.5]
-      fail-fast: false
-    runs-on: ubuntu-latest
-    env:
-      DOCKER_IMAGE: adi611/docker-centos7-slurm:23.02.1
-    steps:
-    - name: Disable etelemetry
-      run: echo "NO_ET=TRUE" >> $GITHUB_ENV
-    - uses: actions/checkout@v4
-    - name: Fetch tags
-      run: git fetch --prune --unshallow
-    - name: Pull docker image
-      run: |
-        docker pull $DOCKER_IMAGE
-        # Have image running in the background
-        docker run `bash <(curl -s https://codecov.io/env)` -itd -h slurmctl --cap-add sys_admin -d --name slurm -v `pwd`:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE
-    - name: Display previous jobs with sacct
-      run: |
-        echo "Allowing ports/daemons time to start" && sleep 10
-        docker exec slurm bash -c "sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'"
-        docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null
-        if [ $? -ne 0 ]; then
-            echo "Slurm docker image error"
-            exit 1
-        fi
-    - name: Setup Python
-      run: |
-        docker exec slurm bash -c "echo $NO_ET"
-        docker exec slurm bash -c "ls -la && echo list top level dir"
-        docker exec slurm bash -c "ls -la /pydra && echo list pydra dir"
-        if [[ "${{ matrix.python-version }}" == "3.11.5" ]]; then
-            docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
-        fi
-        docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
-        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.engine.__version__)'"
-    - name: Run pytest
-      run: |
-        docker exec slurm bash -c "pytest --color=yes -vs -n auto --psij=slurm --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/engine/tests/test_workflow.py /pydra/pydra/engine/tests/test_submitter.py /pydra/pydra/engine/tests/test_node_task.py -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"
-    - name: Upload to codecov
-      run: |
-        docker exec slurm bash -c "pip install urllib3==1.26.6"
-        docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests"
-        docker rm -f slurm
-
-  test-singularity:
-    needs: ['build']
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        python-version: ['3.11', '3.12', '3.13']
-      fail-fast: False
-    steps:
-      - name: Set env
-        run: |
-          echo "RELEASE_VERSION=v3.7.1" >> $GITHUB_ENV
-          echo "NO_ET=TRUE" >> $GITHUB_ENV
-      - name: Setup Singularity
-        uses: actions/checkout@v4
-        with:
-          repository: hpcng/singularity
-          ref: 'v3.7.1'
-          path: 'singularity'
-      - name: Setup GO
-        uses: actions/setup-go@v5
-        with:
-          go-version: '^1.13'
-      - name: Install OS deps
-        run: |
-          sudo apt-get update
-          sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.1 \
-          libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y
-      - name: Build
-        run: |
-          cd singularity
-          ./mconfig --without-suid -p /usr/local/
-          make -C ./builddir
-          sudo make -C ./builddir install
-          cd ..
-      - name: Echo singularity version
-        run: |
-          echo ${{ github.ref }}
-          singularity --version
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Update build tools
-        run: python -m pip install --upgrade pip
-      - name: Checkout Pydra repo
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository }}
-      - name: Fetch tags
-        run: git fetch --prune --unshallow
-      - name: Install pydra (test)
-        run: pip install -e ".[test]"
-      - name: Pytest
-        run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py pydra/engine/tests/test_environments.py
+          pytest -v --dask pydra/engine/test_submitter.py --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
       - name: Upload to codecov
         run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW
 
   test-slurm:
     needs: ['build']
     strategy:
       matrix:
-        python-version: [3.11.5]
+        python-version: [3.11.7]
       fail-fast: false
     runs-on: ubuntu-latest
     env:
@@ -276,10 +171,10 @@ jobs:
             docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
         fi
         docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
-        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra.engine; print(pydra.engine.__version__)'"
+        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.engine.__version__)'"
     - name: Run pytest
       run: |
-        docker exec slurm bash -c "pytest /pydra/pydra/engine/tests/test_workflow.py /pydra/pydra/engine/tests/test_submitter.py /pydra/pydra/engine/tests/test_node_task.py --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"
+        docker exec slurm bash -c "pytest /pydra/pydra/engine/tests/test_submitter.py --with-psij --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/"
     - name: Upload to codecov
       run: |
         docker exec slurm bash -c "pip install urllib3==1.26.6"
@@ -357,7 +252,7 @@ jobs:
         path: docs/build/html
 
   deploy:
-    needs: [build-docs, test, test-dask, test-psijlocal, test-psijslurm, test-singularity, test-slurm]
+    needs: [build-docs, test, test-dask, test-slurm]
     runs-on: ubuntu-latest
     steps:
     - name: Download dist

docs/source/explanation/environments.rst

@@ -7,7 +7,7 @@ construction, and allows tasks to be run in environments that are isolated from
 host system, and that have specific software dependencies.
 
 The environment a task runs within is specified by the ``environment`` argument passed
-to the execution call (e.g. ``my_task(plugin="cf", environment="docker")``) or in the
+to the execution call (e.g. ``my_task(worker="cf", environment="docker")``) or in the
 ``workflow.add()`` call in workflow constructors.
 
 Specifying at execution

docs/tst.py

@@ -8,69 +8,35 @@
 def pytest_addoption(parser):
     parser.addoption("--dask", action="store_true", help="run all combinations")
     parser.addoption(
-        "--psij",
-        action="store",
-        help="run with psij subtype plugin",
-        choices=["local", "slurm"],
+        "--with-psij",
+        action="store_true",
+        help="run with psij workers in test matrix",
     )
 
 
 def pytest_generate_tests(metafunc):
-    if "plugin_dask_opt" in metafunc.fixturenames:
-        if bool(shutil.which("sbatch")):
-            Plugins = ["slurm"]
-        else:
-            Plugins = ["debug", "cf"]
-        try:
-            if metafunc.config.getoption("dask"):
-                Plugins = ["dask"]
-        except ValueError:
-            # Called as --pyargs, so --dask isn't available
-            pass
-        try:
-            if metafunc.config.getoption("psij"):
-                Plugins = ["psij-" + metafunc.config.getoption("psij")]
-        except ValueError:
-            pass
-        metafunc.parametrize("plugin_dask_opt", Plugins)
+    if "worker" in metafunc.fixturenames:
+        metafunc.parametrize("worker", ["debug", "cf"])
 
-    if "plugin" in metafunc.fixturenames:
-        use_dask = False
+    if "any_worker" in metafunc.fixturenames:
+        available_workers = ["debug", "cf"]
         try:
             use_dask = metafunc.config.getoption("dask")
         except ValueError:
-            pass
-        if use_dask:
-            Plugins = []
-        elif bool(shutil.which("sbatch")):
-            Plugins = ["slurm"]
-        else:
-            Plugins = ["debug", "cf"]
+            use_dask = False
         try:
-            if metafunc.config.getoption("psij"):
-                Plugins = ["psij-" + metafunc.config.getoption("psij")]
+            use_psij = metafunc.config.getoption("with-psij")
         except ValueError:
-            pass
-        metafunc.parametrize("plugin", Plugins)
-
-    if "plugin_parallel" in metafunc.fixturenames:
-        use_dask = False
-        try:
-            use_dask = metafunc.config.getoption("dask")
-        except ValueError:
-            pass
+            use_psij = False
         if use_dask:
-            Plugins = []
-        elif bool(shutil.which("sbatch")):
-            Plugins = ["slurm"]
-        else:
-            Plugins = ["cf"]
-        try:
-            if metafunc.config.getoption("psij"):
-                Plugins = ["psij-" + metafunc.config.getoption("psij")]
-        except ValueError:
-            pass
-        metafunc.parametrize("plugin_parallel", Plugins)
+            available_workers.append("dask")
+        if bool(shutil.which("sbatch")):
+            available_workers.append("slurm")
+            if use_psij:
+                available_workers.append("psij-slurm")
+        if use_psij:
+            available_workers.append("psij-local")
+        metafunc.parametrize("any_worker", available_workers)
 
 
 # For debugging in IDE's don't catch raised exceptions and let the IDE

pydra/conftest.py

@@ -139,7 +139,7 @@ def __init__(
                     worker_cls = WORKERS[self.worker_name]
                 except KeyError:
                     raise NotImplementedError(
-                        f"No worker for '{self.worker_name}' plugin"
+                        f"No worker for '{self.worker_name}' worker"
                     )
             else:
                 try:

pydra/engine/submitter.py

@@ -23,12 +23,12 @@
     "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"]
 )
 @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter])
-def test_boutiques_1(maskfile, plugin, results_function, tmpdir, data_tests_dir):
+def test_boutiques_1(maskfile, worker, results_function, tmpdir, data_tests_dir):
     """simple task to run fsl.bet using BoshTask"""
     btask = boutiques.define(zenodo_id="1482743")
     btask.infile = data_tests_dir / "test.nii.gz"
     btask.maskfile = maskfile
-    res = btask(plugin, cache_dir=tmpdir)
+    res = btask(worker, cache_dir=tmpdir)
 
     assert res.output.return_code == 0
 
@@ -92,7 +92,7 @@ def test_boutiques_spec_2(data_tests_dir):
 @pytest.mark.parametrize(
     "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"]
 )
-def test_boutiques_wf_1(maskfile, plugin, tmpdir, infile):
+def test_boutiques_wf_1(maskfile, worker, tmpdir, infile):
     """wf with one task that runs fsl.bet using BoshTask"""
 
     @workflow.define
@@ -107,7 +107,7 @@ def Workflow(maskfile, infile):
         return bet.outfile
 
     wf = Workflow(maskfile=maskfile, infile=infile)
-    wf(plugin=plugin, cache_dir=tmpdir)
+    wf(worker=worker, cache_dir=tmpdir)
 
     res = wf.result()
     assert res.output.outfile.name == "test_brain.nii.gz"
@@ -121,7 +121,7 @@ def Workflow(maskfile, infile):
 @pytest.mark.parametrize(
     "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"]
 )
-def test_boutiques_wf_2(maskfile, plugin, tmpdir, infile):
+def test_boutiques_wf_2(maskfile, worker, tmpdir, infile):
     """wf with two BoshTasks (fsl.bet and fsl.stats) and one ShellTask"""
 
     @workflow.define(outputs=["outfile_bet", "out_stat", "out"])
@@ -143,7 +143,7 @@ def Workflow(maskfile, infile):
         cat = workflow.add(shell.define("cat <file>")(file=stat.output))
         return bet.outfile, stat.output, cat.stdout
 
-    res = Workflow(maskfile=maskfile, infile=infile)(plugin=plugin, cache_dir=tmpdir)
+    res = Workflow(maskfile=maskfile, infile=infile)(worker=worker, cache_dir=tmpdir)
     assert res.output.outfile_bet.name == "test_brain.nii.gz"
     assert res.output.outfile_bet.exists()