split out dask and psij workers into separate plugin packages

Author: tclose

.github/workflows/ci-cd.yml

@@ -80,38 +80,6 @@ jobs:
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-workers:
-    needs: ['build']
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ['3.11', '3.12', '3.13']
-        worker: ["psij-local", "dask"]
-      fail-fast: false
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository }}
-      - name: Fetch tags
-        run: git fetch --prune --unshallow
-      - name: Setup Python version ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install package
-        run: |
-          pip install -e ".[test,dask,psij]"
-      - name: Run tests for Dask
-        run: |
-          pytest -v pydra/engine/tests/test_submitter.py --only-worker=${{ matrix.worker }} --rootdir pydra --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
-        with:
-          fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
-
   test-singularity:
     needs: ['build']
     runs-on: ubuntu-22.04
@@ -177,7 +145,6 @@ jobs:
     strategy:
       matrix:
         python-version: [3.11.5]
-        worker: ["slurm", "psij-slurm"]
       fail-fast: false
     runs-on: ubuntu-latest
     env:
@@ -214,7 +181,7 @@ jobs:
         docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.utils.__version__)'"
     - name: Run pytest
       run: |
-        docker exec slurm bash -c "pytest /pydra/pydra/engine/tests/test_submitter.py --rootdir /pydra/pydra --only-worker=${{ matrix.worker }} --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml"
+        docker exec slurm bash -c "pytest /pydra/pydra/engine/tests/test_submitter.py --rootdir /pydra/pydra --only-worker=slurm --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml"
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2
       with:
@@ -373,7 +340,7 @@ jobs:
         path: docs/build/html
 
   deploy:
-    needs: [build-docs, test, test-workers, test-singularity, test-slurm]
+    needs: [build-docs, test, test-singularity, test-slurm]
     runs-on: ubuntu-latest
     steps:
     - name: Download dist

pydra/conftest.py

@@ -7,11 +7,6 @@
 
 def pytest_addoption(parser):
     parser.addoption("--with-dask", action="store_true", help="run all combinations")
-    parser.addoption(
-        "--with-psij",
-        action="store_true",
-        help="run with psij workers in test matrix",
-    )
     parser.addoption(
         "--only-worker",
         help="only run tests with provided worker",
@@ -29,10 +24,6 @@ def pytest_generate_tests(metafunc):
             with_dask = metafunc.config.getoption("with_dask")
         except ValueError:
             with_dask = False
-        try:
-            with_psij = metafunc.config.getoption("with_psij")
-        except ValueError:
-            with_psij = False
         try:
             only_worker = metafunc.config.getoption("only_worker")
         except ValueError:
@@ -43,10 +34,6 @@ def pytest_generate_tests(metafunc):
                 available_workers.append("dask")
             if bool(shutil.which("sbatch")):
                 available_workers.append("slurm")
-                if with_psij:
-                    available_workers.append("psij-slurm")
-            if with_psij:
-                available_workers.append("psij-local")
         else:
             available_workers = [only_worker]
         # Set the available workers as a parameter to the

pydra/workers/cf.py

@@ -5,7 +5,7 @@
 import concurrent.futures as cf
 import logging
 from pydra.engine.job import Job
-from . import base
+from pydra.workers import base
 
 logger = logging.getLogger("pydra.worker")
 

pydra/workers/dask.py

@@ -1,7 +1,7 @@
 import typing as ty
 import logging
 from pydra.engine.job import Job
-from . import base
+from pydra.workers import base
 
 if ty.TYPE_CHECKING:
     from pydra.engine.result import Result

pydra/workers/debug.py

@@ -10,7 +10,7 @@
 import random
 import logging
 from pydra.engine.job import Job, save, load_job
-from . import base
+from pydra.workers import base
 
 logger = logging.getLogger("pydra.worker")
 

pydra/workers/psij_base.py

@@ -9,7 +9,7 @@
 import logging
 import attrs
 from pydra.engine.job import Job, save
-from . import base
+from pydra.workers import base
 
 
 logger = logging.getLogger("pydra.worker")