Add Ruff to cluster_tools (#1023)

* added ruff to cluster_tools

* auto-format

* manually fixed linting errors

* removed pylint coments

* update changelog

* fix typechecking

* fixed linting

Author: hotzenklotz

README.md

@@ -3,7 +3,7 @@
 [![Supported Python Versions](https://img.shields.io/pypi/pyversions/webknossos.svg)](https://pypi.python.org/pypi/webknossos)
 [![Build Status](https://img.shields.io/github/actions/workflow/status/scalableminds/webknossos-libs/.github/workflows/ci.yml?branch=master)](https://github.com/scalableminds/webknossos-libs/actions?query=workflow%3A%22CI%22)
 [![Documentation](https://img.shields.io/badge/docs-passing-brightgreen.svg)](https://docs.webknossos.org/webknossos-py/index.html)
-[![Code Style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Code Style](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://docs.astral.sh/ruff/)
 
 <img align="right" src="https://static.webknossos.org/logos/webknossos-icon-only.svg" alt="WEBKNOSSOS Logo" width="100" height="100"/>
 

cluster_tools/Changelog.md

@@ -14,6 +14,7 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 ### Added
 
 ### Changed
+- Replaced black+isort formatters and pylint linter with ruff. [#1023](https://github.com/scalableminds/webknossos-libs/pull/1023)
 
 ### Fixed
 

cluster_tools/README.md

@@ -1,6 +1,8 @@
 # Cluster Tools
 
 [![Build Status](https://img.shields.io/github/actions/workflow/status/scalableminds/webknossos-libs/.github/workflows/ci.yml?branch=master)](https://github.com/scalableminds/webknossos-libs/actions?query=workflow%3A%22CI%22)
+[![Code Style](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://docs.astral.sh/ruff/)
+
 
 This package provides python `Executor` classes for distributing tasks on a slurm cluster or via multi processing.
 

cluster_tools/cluster_tools/__init__.py

@@ -7,10 +7,10 @@
 from cluster_tools.executors.pickle_ import PickleExecutor
 from cluster_tools.executors.sequential import SequentialExecutor
 from cluster_tools.schedulers.cluster_executor import (
-    ClusterExecutor,
-    RemoteOutOfMemoryException,
-    RemoteResourceLimitException,
-    RemoteTimeLimitException,
+    ClusterExecutor,  # noqa:  F401 `cluster_tools.schedulers.cluster_executor.ClusterExecutor` imported but unused;
+    RemoteOutOfMemoryException,  # noqa:  F401 `cluster_tools.schedulers.cluster_executor.ClusterExecutor` imported but unused;
+    RemoteResourceLimitException,  # noqa:  F401 `cluster_tools.schedulers.cluster_executor.ClusterExecutor` imported but unused;
+    RemoteTimeLimitException,  # noqa:  F401 `cluster_tools.schedulers.cluster_executor.ClusterExecutor` imported but unused;
 )
 from cluster_tools.schedulers.kube import KubernetesExecutor
 from cluster_tools.schedulers.pbs import PBSExecutor
@@ -54,53 +54,45 @@ def _test_valid_multiprocessing() -> None:
 
 
 @overload
-def get_executor(environment: Literal["slurm"], **kwargs: Any) -> SlurmExecutor:
-    ...
+def get_executor(environment: Literal["slurm"], **kwargs: Any) -> SlurmExecutor: ...
 
 
 @overload
-def get_executor(environment: Literal["pbs"], **kwargs: Any) -> PBSExecutor:
-    ...
+def get_executor(environment: Literal["pbs"], **kwargs: Any) -> PBSExecutor: ...
 
 
 @overload
 def get_executor(
     environment: Literal["kubernetes"], **kwargs: Any
-) -> KubernetesExecutor:
-    ...
+) -> KubernetesExecutor: ...
 
 
 @overload
-def get_executor(environment: Literal["dask"], **kwargs: Any) -> DaskExecutor:
-    ...
+def get_executor(environment: Literal["dask"], **kwargs: Any) -> DaskExecutor: ...
 
 
 @overload
 def get_executor(
     environment: Literal["multiprocessing"], **kwargs: Any
-) -> MultiprocessingExecutor:
-    ...
+) -> MultiprocessingExecutor: ...
 
 
 @overload
 def get_executor(
     environment: Literal["sequential"], **kwargs: Any
-) -> SequentialExecutor:
-    ...
+) -> SequentialExecutor: ...
 
 
 @overload
 def get_executor(
     environment: Literal["debug_sequential"], **kwargs: Any
-) -> DebugSequentialExecutor:
-    ...
+) -> DebugSequentialExecutor: ...
 
 
 @overload
 def get_executor(
     environment: Literal["test_pickling"], **kwargs: Any
-) -> PickleExecutor:
-    ...
+) -> PickleExecutor: ...
 
 
 def get_executor(environment: str, **kwargs: Any) -> "Executor":

cluster_tools/cluster_tools/_utils/call.py

@@ -25,9 +25,7 @@ def call(command: str, stdin: Optional[str] = None) -> Tuple[str, str, int]:
 class CommandError(Exception):
     """Raised when a shell command exits abnormally."""
 
-    def __init__(
-        self, command: str, code: int, stderr: str
-    ):  # pylint: disable=super-init-not-called
+    def __init__(self, command: str, code: int, stderr: str):
         self.command = command
         self.code = code
         self.stderr = stderr

cluster_tools/cluster_tools/_utils/multiprocessing_logging_handler.py

@@ -52,7 +52,7 @@ def _receive(self) -> None:
                 record = self.queue.get(timeout=0.2)
                 if record is not None:
                     self.wrapped_handler.emit(record)
-            except (KeyboardInterrupt, SystemExit):  # pylint: disable=try-except-raise
+            except (KeyboardInterrupt, SystemExit):
                 raise
             # The following errors pop up quite often.
             # It seems that they can be safely ignored, though.

cluster_tools/cluster_tools/_utils/pickling.py

@@ -46,7 +46,5 @@ def find_class(self, module: str, name: str) -> Any:
 @warn_after("pickle.load", WARNING_TIMEOUT)
 def load(f: BinaryIO, custom_main_path: Optional[str] = None) -> Any:
     unpickler = _RenameUnpickler(f)
-    unpickler.custom_main_path = (  # pylint: disable=attribute-defined-outside-init
-        custom_main_path
-    )
+    unpickler.custom_main_path = custom_main_path
     return unpickler.load()

cluster_tools/cluster_tools/executor_protocol.py

@@ -20,40 +20,35 @@
 
 class Executor(Protocol, ContextManager["Executor"]):
     @classmethod
-    def as_completed(cls, futures: List["Future[_T]"]) -> Iterator["Future[_T]"]:
-        ...
+    def as_completed(cls, futures: List["Future[_T]"]) -> Iterator["Future[_T]"]: ...
 
     def submit(
         self,
         __fn: Callable[_P, _T],
         /,
         *args: _P.args,
         **kwargs: _P.kwargs,
-    ) -> "Future[_T]":
-        ...
+    ) -> "Future[_T]": ...
 
-    def map_unordered(self, fn: Callable[[_S], _T], args: Iterable[_S]) -> Iterator[_T]:
-        ...
+    def map_unordered(
+        self, fn: Callable[[_S], _T], args: Iterable[_S]
+    ) -> Iterator[_T]: ...
 
     def map_to_futures(
         self,
         fn: Callable[[_S], _T],
         args: Iterable[_S],
         output_pickle_path_getter: Optional[Callable[[_S], PathLike]] = None,
-    ) -> List["Future[_T]"]:
-        ...
+    ) -> List["Future[_T]"]: ...
 
     def map(
         self,
         fn: Callable[[_S], _T],
         iterables: Iterable[_S],
         timeout: Optional[float] = None,
         chunksize: Optional[int] = None,
-    ) -> Iterator[_T]:
-        ...
+    ) -> Iterator[_T]: ...
 
-    def forward_log(self, fut: "Future[_T]") -> _T:
-        ...
+    def forward_log(self, fut: "Future[_T]") -> _T: ...
 
-    def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
-        ...
+    def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None: ...

cluster_tools/cluster_tools/executors/dask.py

@@ -239,7 +239,9 @@ def result_generator() -> Iterator:
     def map_to_futures(
         self,
         fn: Callable[[_S], _T],
-        args: Iterable[_S],  # TODO change: allow more than one arg per call
+        args: Iterable[
+            _S
+        ],  # TODO change: allow more than one arg per call # noqa FIX002 Line contains TODO
         output_pickle_path_getter: Optional[Callable[[_S], os.PathLike]] = None,
     ) -> List["Future[_T]"]:
         if output_pickle_path_getter is not None:
@@ -286,13 +288,12 @@ def handle_kill(
         self.client.cancel(list(self.pending_futures))
 
         if (
-            existing_sigint_handler  # pylint: disable=comparison-with-callable
-            != signal.default_int_handler
+            existing_sigint_handler != signal.default_int_handler
             and callable(existing_sigint_handler)  # Could also be signal.SIG_IGN
         ):
             existing_sigint_handler(signum, frame)
 
-    def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
+    def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:  # noqa: ARG002 Unused method argument: `cancel_futures`
         if wait:
             for fut in list(self.pending_futures):
                 fut.result()

cluster_tools/cluster_tools/executors/multiprocessing_.py

@@ -246,7 +246,9 @@ def result_generator() -> Iterator:
     def map_to_futures(
         self,
         fn: Callable[[_S], _T],
-        args: Iterable[_S],  # TODO change: allow more than one arg per call
+        args: Iterable[
+            _S
+        ],  # TODO change: allow more than one arg per call #noqa: FIX002 Line contains TODO
         output_pickle_path_getter: Optional[Callable[[_S], os.PathLike]] = None,
     ) -> List["Future[_T]"]:
         if output_pickle_path_getter is not None: