deprecate dask and kubernetes schedulers (#1375)

* deprecate dask and kubernetes schedulers

* changelog

Author: normanrz

.github/workflows/ci.yml

@@ -124,74 +124,6 @@ jobs:
           PYTEST_EXECUTORS=multiprocessing,sequential,multiprocessing_with_pickling,sequential_with_pickling \
             uv run --frozen python -m pytest -sv test_all.py test_multiprocessing.py
 
-  cluster_tools_kubernetes:
-    needs: changes
-    if: ${{ needs.changes.outputs.cluster_tools == 'true' }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        python-version: ["3.13", "3.12", "3.11", "3.10"]
-    defaults:
-      run:
-        working-directory: cluster_tools
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: true
-          cache-dependency-glob: "cluster_tools/uv.lock"
-      - name: Set up Python ${{ matrix.python-version }}
-        run: uv python install ${{ matrix.python-version }}
-      - name: Setup Kubernetes-in-Docker
-        run: |
-          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
-          chmod +x ./kind
-          sed -i "s#__PATH__#$(pwd)#g" tests/cluster-config.yaml
-          ./kind create cluster --config=tests/cluster-config.yaml
-          ./kind export kubeconfig
-
-          docker build \
-            --build-arg PYTHON_VERSION=${{ matrix.python-version }} \
-            -f tests/Dockerfile \
-            -t scalableminds/cluster-tools:latest \
-            .
-          ./kind load docker-image scalableminds/cluster-tools:latest
-      - name: Install dependencies (without docker)
-        run: uv sync --all-extras --frozen
-      - name: "Run Kubernetes"
-        run: |
-          cd tests
-          PYTEST_EXECUTORS=kubernetes uv run --frozen python -m pytest -sv test_all.py test_kubernetes.py
-
-  cluster_tools_dask:
-    needs: changes
-    if: ${{ needs.changes.outputs.cluster_tools == 'true' }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        python-version: ["3.13", "3.12", "3.11", "3.10"]
-    defaults:
-      run:
-        working-directory: cluster_tools
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: true
-          cache-dependency-glob: "cluster_tools/uv.lock"
-      - name: Set up Python ${{ matrix.python-version }}
-        run: uv python install ${{ matrix.python-version }}
-      - name: Install dependencies (without docker)
-        run: uv sync --all-extras --frozen
-      - name: "Run Dask"
-        run: |
-          cd tests
-          PYTEST_EXECUTORS=dask uv run --frozen python -m pytest -sv test_all.py test_dask.py
-
   webknossos_linux:
     needs: changes
     if: |
@@ -347,8 +279,6 @@ jobs:
     needs:
       - cluster_tools_slurm
       - cluster_tools_multiprocessing
-      - cluster_tools_kubernetes
-      - cluster_tools_dask
       - webknossos_linux
       - webknossos_windows
     runs-on: ubuntu-latest
@@ -406,8 +336,6 @@ jobs:
     needs:
       - cluster_tools_slurm
       - cluster_tools_multiprocessing
-      - cluster_tools_kubernetes
-      - cluster_tools_dask
       - webknossos_linux
       - webknossos_windows
     if: |
@@ -443,8 +371,6 @@ jobs:
 
   complete:
     needs:
-      - cluster_tools_dask
-      - cluster_tools_kubernetes
       - cluster_tools_multiprocessing
       - cluster_tools_slurm
       - webknossos_linux

cluster_tools/Changelog.md

@@ -10,6 +10,7 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v2.5.0...HEAD)
 
 ### Breaking Changes
+- Deprecate the `dask` and `kubernetes` executors. They will be removed in a future version. [#1375](https://github.com/scalableminds/webknossos-libs/pull/1375)
 
 ### Added
 

cluster_tools/cluster_tools/executors/dask.py

@@ -14,6 +14,7 @@
     TypeVar,
     cast,
 )
+from warnings import warn
 from weakref import ReferenceType, ref
 
 from typing_extensions import ParamSpec
@@ -117,6 +118,12 @@ def __init__(
             )
             exit()
 
+        warn(
+            "Dask support is deprecated and will be removed in a future version.",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+
         self.client = client
         self.pending_futures = set()
         self.job_resources = job_resources

cluster_tools/cluster_tools/schedulers/kube.py

@@ -8,6 +8,7 @@
 from pathlib import Path
 from typing import Any, Literal
 from uuid import uuid4
+from warnings import warn
 
 from cluster_tools.schedulers.cluster_executor import ClusterExecutor
 
@@ -58,6 +59,12 @@ def __init__(
             )
             exit()
 
+        warn(
+            "Kubernetes support is deprecated and will be removed in a future version.",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+
         super().__init__(
             debug=debug,
             keep_logs=keep_logs,

cluster_tools/test.sh

@@ -4,4 +4,4 @@ set -eEuo pipefail
   cd tests
   PYTEST_EXECUTORS=multiprocessing,multiprocessing_with_pickling,sequential uv run --frozen python -m pytest -sv test_all.py test_multiprocessing.py
 
-  echo "Tests for the kubernetes, dask and SLURM executors are only run in the CI"
+  echo "Tests for the SLURM executors are only run in the CI"