From f5acf41fe1d6906b7e4c48ae71e5739eb44a479f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 15 May 2025 16:05:41 +0200 Subject: [PATCH 01/46] now we also test the scheduler preload --- .../simcore_service_dask_sidecar/scheduler.py | 4 +-- .../task_life_cycle_scheduler_plugin.py | 35 +++++++++++++++++++ services/dask-sidecar/tests/unit/conftest.py | 1 + .../dask-sidecar/tests/unit/test_scheduler.py | 9 +++++ 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py create mode 100644 services/dask-sidecar/tests/unit/test_scheduler.py diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py index 4127fca2528b..721a86d93cd3 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py @@ -22,6 +22,6 @@ async def dask_setup(scheduler: distributed.Scheduler) -> None: print_dask_scheduler_banner() -async def dask_teardown(_worker: distributed.Worker) -> None: - with log_context(_logger, logging.INFO, "Tear down dask scheduler"): +async def dask_teardown(scheduler: distributed.Scheduler) -> None: + with log_context(_logger, logging.INFO, f"Tear down dask {scheduler.address}"): ... diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py new file mode 100644 index 000000000000..f27275889b71 --- /dev/null +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py @@ -0,0 +1,35 @@ +from typing import Any + +from dask.typing import Key +from distributed import SchedulerPlugin +from distributed.scheduler import TaskStateState + + +class SchedulerLifecyclePlugin(SchedulerPlugin): + def __init__(self) -> None: + self.scheduler = None + + def add_task(self, key, **kwargs): + """Task published to cluster""" + self.scheduler.log_event( + "task-published", + {"key": key, "timestamp": time.time(), "client": kwargs.get("client")}, + ) + + def transition( + self, + key: Key, + start: TaskStateState, + finish: TaskStateState, + **kwargs: Any, + ): + """State transitions""" + if finish in ("waiting", "processing", "memory", "erred"): + self.scheduler.log_event( + f"task-{finish}", + { + "key": key, + "worker": kwargs.get("worker"), + "duration": time.time() - self.scheduler.tasks[key].start_time, + }, + ) diff --git a/services/dask-sidecar/tests/unit/conftest.py b/services/dask-sidecar/tests/unit/conftest.py index 944e142c1298..2e3fb246f884 100644 --- a/services/dask-sidecar/tests/unit/conftest.py +++ b/services/dask-sidecar/tests/unit/conftest.py @@ -117,6 +117,7 @@ def local_cluster(app_environment: EnvVarsDict) -> Iterator[distributed.LocalClu with distributed.LocalCluster( worker_class=distributed.Worker, resources={"CPU": 10, "GPU": 10}, + scheduler_kwargs={"preload": "simcore_service_dask_sidecar.scheduler"}, preload="simcore_service_dask_sidecar.worker", ) as cluster: assert cluster diff --git a/services/dask-sidecar/tests/unit/test_scheduler.py b/services/dask-sidecar/tests/unit/test_scheduler.py new file mode 100644 index 000000000000..0a2c5dce032f --- /dev/null +++ b/services/dask-sidecar/tests/unit/test_scheduler.py @@ -0,0 +1,9 @@ +import distributed + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +def test_scheduler(dask_client: distributed.Client) -> None: + assert True From 6963a8c94ef101fdbcf81894352781537451dd11 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 15 May 2025 16:19:04 +0200 Subject: [PATCH 02/46] improve logs --- .../simcore_service_dask_sidecar/scheduler.py | 9 +++- .../task_life_cycle_scheduler_plugin.py | 45 +++++++++++++------ .../simcore_service_dask_sidecar/worker.py | 4 +- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py index 721a86d93cd3..31ca65b944f0 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py @@ -5,6 +5,9 @@ from ._meta import print_dask_scheduler_banner from .settings import ApplicationSettings +from .task_life_cycle_scheduler_plugin import ( + TaskLifecycleSchedulerPlugin, +) from .utils.logs import setup_app_logging _logger = logging.getLogger(__name__) @@ -19,9 +22,13 @@ async def dask_setup(scheduler: distributed.Scheduler) -> None: with log_context(_logger, logging.INFO, "Launch dask scheduler"): _logger.info("app settings: %s", settings.model_dump_json(indent=1)) + + scheduler.add_plugin(TaskLifecycleSchedulerPlugin()) print_dask_scheduler_banner() async def dask_teardown(scheduler: distributed.Scheduler) -> None: - with log_context(_logger, logging.INFO, f"Tear down dask {scheduler.address}"): + with log_context( + _logger, logging.INFO, f"Tear down dask scheduler at {scheduler.address}" + ): ... diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py index f27275889b71..9b33ccd9e902 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py @@ -1,35 +1,54 @@ +import logging from typing import Any from dask.typing import Key -from distributed import SchedulerPlugin +from distributed import Scheduler, SchedulerPlugin from distributed.scheduler import TaskStateState +from servicelib.logging_utils import log_context +_logger = logging.getLogger(__name__) -class SchedulerLifecyclePlugin(SchedulerPlugin): + +class TaskLifecycleSchedulerPlugin(SchedulerPlugin): def __init__(self) -> None: - self.scheduler = None + with log_context( + _logger, + logging.INFO, + "TaskLifecycleSchedulerPlugin init", + ): + self.scheduler = None - def add_task(self, key, **kwargs): - """Task published to cluster""" - self.scheduler.log_event( - "task-published", - {"key": key, "timestamp": time.time(), "client": kwargs.get("client")}, - ) + async def start(self, scheduler: Scheduler) -> None: + with log_context( + _logger, + logging.INFO, + "TaskLifecycleSchedulerPlugin start", + ): + self.scheduler = scheduler def transition( self, key: Key, start: TaskStateState, finish: TaskStateState, + *args: Any, + stimulus_id: str, **kwargs: Any, ): - """State transitions""" - if finish in ("waiting", "processing", "memory", "erred"): + # Start state: one of released, waiting, processing, memory, error + with log_context( + _logger, + logging.INFO, + f"Task {key} transition from {start} to {finish} due to {stimulus_id=}", + ): + assert self.scheduler # nosec self.scheduler.log_event( - f"task-{finish}", + f"task-lifecycle-{key}", { "key": key, "worker": kwargs.get("worker"), - "duration": time.time() - self.scheduler.tasks[key].start_time, + "start": start, + "finish": finish, + "stimulus_id": stimulus_id, }, ) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py index de3ecd6d66cc..3b3c07fa8ab2 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py @@ -83,7 +83,9 @@ async def dask_setup(worker: distributed.Worker) -> None: async def dask_teardown(worker: distributed.Worker) -> None: - with log_context(_logger, logging.INFO, f"tear down dask {worker.address}"): + with log_context( + _logger, logging.INFO, f"tear down dask worker at {worker.address}" + ): ... From 4ca2bd3321f2437a81485c6212db457058b945ad Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 15 May 2025 16:29:59 +0200 Subject: [PATCH 03/46] show manual intervention needed if started since >2 minutes --- .../computational-clusters/autoscaled_monitor/ssh.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/maintenance/computational-clusters/autoscaled_monitor/ssh.py b/scripts/maintenance/computational-clusters/autoscaled_monitor/ssh.py index 229dab0c3f40..e94fed195e42 100644 --- a/scripts/maintenance/computational-clusters/autoscaled_monitor/ssh.py +++ b/scripts/maintenance/computational-clusters/autoscaled_monitor/ssh.py @@ -265,7 +265,11 @@ def _needs_manual_intervention( user_id=containers[0].user_id, project_id=containers[0].project_id, created_at=containers[0].created_at, - needs_manual_intervention=_needs_manual_intervention(containers), + needs_manual_intervention=_needs_manual_intervention(containers) + and ( + (containers[0].created_at - arrow.utcnow().datetime) + > datetime.timedelta(minutes=2) + ), containers=[c.name for c in containers], service_name=containers[0].service_name, service_version=containers[0].service_version, From 2107619f3b655a3ccf2a6a4e4bed581aa44ab170 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 15 May 2025 17:05:03 +0200 Subject: [PATCH 04/46] add some better reason --- .../dask-sidecar/src/simcore_service_dask_sidecar/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py index 3b3c07fa8ab2..7344326a409b 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py @@ -76,7 +76,7 @@ async def dask_setup(worker: distributed.Worker) -> None: RabbitMQPlugin(settings.DASK_SIDECAR_RABBITMQ), catch_errors=False ) except Exception: - await worker.close() + await worker.close(reason="failed to add RabbitMQ plugin") raise print_dask_sidecar_banner() From 6224bf8073625762691895480e64d14da477d7f5 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 15 May 2025 18:13:27 +0200 Subject: [PATCH 05/46] added life cycle plugins --- .../task_life_cycle_worker_plugin.py | 55 +++++++++++++++++++ .../simcore_service_dask_sidecar/worker.py | 8 +++ .../tests/unit/test_rabbitmq_plugin.py | 21 +++++++ .../dask-sidecar/tests/unit/test_scheduler.py | 31 ++++++++++- 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py new file mode 100644 index 000000000000..d384c02cda58 --- /dev/null +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py @@ -0,0 +1,55 @@ +import logging +from collections.abc import Awaitable +from typing import Any + +from dask.typing import Key +from distributed import Worker, WorkerPlugin +from distributed.scheduler import TaskStateState +from servicelib.logging_utils import log_context + +_logger = logging.getLogger(__name__) + + +class TaskLifecycleWorkerPlugin(WorkerPlugin): + def __init__(self) -> None: + with log_context( + _logger, + logging.INFO, + "TaskLifecycleWorkerPlugin init", + ): + self.worker = None + + def setup(self, worker: Worker) -> Awaitable[None]: + async def _() -> None: + with log_context( + _logger, + logging.INFO, + "TaskLifecycleWorkerPlugin start", + ): + self.worker = worker + + return _() + + def transition( + self, + key: Key, + start: TaskStateState, + finish: TaskStateState, + **kwargs: Any, + ): + # Start state: one of released, waiting, processing, memory, error + with log_context( + _logger, + logging.INFO, + f"Task {key} transition from {start} to {finish}", + ): + assert self.worker # nosec + self.worker.log_event( + f"task-lifecycle-{key}", + { + "key": key, + "worker": kwargs.get("worker"), + "start": start, + "finish": finish, + }, + ) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py index 7344326a409b..df6b4a482664 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py @@ -18,6 +18,9 @@ from .computational_sidecar.core import ComputationalSidecar from .rabbitmq_plugin import RabbitMQPlugin from .settings import ApplicationSettings +from .task_life_cycle_worker_plugin import ( + TaskLifecycleWorkerPlugin, +) from .utils.dask import ( TaskPublisher, get_current_task_resources, @@ -78,6 +81,11 @@ async def dask_setup(worker: distributed.Worker) -> None: except Exception: await worker.close(reason="failed to add RabbitMQ plugin") raise + try: + await worker.plugin_add(TaskLifecycleWorkerPlugin(), catch_errors=False) + except Exception: + await worker.close(reason="failed to add TaskLifecycleWorkerPlugin") + raise print_dask_sidecar_banner() diff --git a/services/dask-sidecar/tests/unit/test_rabbitmq_plugin.py b/services/dask-sidecar/tests/unit/test_rabbitmq_plugin.py index de632c818ec8..dd644d64fa06 100644 --- a/services/dask-sidecar/tests/unit/test_rabbitmq_plugin.py +++ b/services/dask-sidecar/tests/unit/test_rabbitmq_plugin.py @@ -3,7 +3,12 @@ # pylint: disable=unused-variable # pylint: disable=no-member +import asyncio +from unittest import mock + import distributed +import pytest +from pytest_mock import MockerFixture # Selection of core and tool services started in this swarm fixture (integration) pytest_simcore_core_services_selection = [ @@ -14,3 +19,19 @@ def test_rabbitmq_plugin_initializes(dask_client: distributed.Client): ... + + +@pytest.fixture +def erroring_rabbitmq_plugin(mocker: MockerFixture) -> mock.Mock: + return mocker.patch( + "simcore_service_dask_sidecar.rabbitmq_plugin.RabbitMQPlugin", + autospec=True, + side_effect=RuntimeError("Pytest: RabbitMQ plugin initialization failed"), + ) + + +async def test_dask_worker_closes_if_plugin_fails_on_start( + erroring_rabbitmq_plugin: mock.Mock, + local_cluster: distributed.LocalCluster, +): + await asyncio.sleep(10) diff --git a/services/dask-sidecar/tests/unit/test_scheduler.py b/services/dask-sidecar/tests/unit/test_scheduler.py index 0a2c5dce032f..7f709cb5c370 100644 --- a/services/dask-sidecar/tests/unit/test_scheduler.py +++ b/services/dask-sidecar/tests/unit/test_scheduler.py @@ -1,4 +1,7 @@ +import time + import distributed +import pytest pytest_simcore_core_services_selection = [ "rabbit", @@ -6,4 +9,30 @@ def test_scheduler(dask_client: distributed.Client) -> None: - assert True + def _some_task() -> int: + time.sleep(1) + return 2 + + def _some_failing_task() -> None: + time.sleep(1) + msg = "Some error" + raise RuntimeError(msg) + + future = dask_client.submit(_some_task) + assert future.result(timeout=10) == 2 + events = dask_client.get_events(f"task-lifecycle-{future.key}") + print("XXXX received events:") + assert events + assert isinstance(events, tuple) + for event in events: + print(f"\t{event}") + + future = dask_client.submit(_some_failing_task) + with pytest.raises(RuntimeError): + future.result(timeout=10) + events = dask_client.get_events(f"task-lifecycle-{future.key}") + print("XXXX received events:") + assert events + assert isinstance(events, tuple) + for event in events: + print(f"\t{event}") From 8d302cb62fbc8b465dabebe0b4582e11c9e4785b Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 07:53:59 +0200 Subject: [PATCH 06/46] renamed --- .../{rabbitmq_plugin.py => rabbitmq_worker_plugin.py} | 2 +- .../src/simcore_service_dask_sidecar/utils/dask.py | 2 +- .../dask-sidecar/src/simcore_service_dask_sidecar/worker.py | 2 +- services/dask-sidecar/tests/unit/test_rabbitmq_plugin.py | 2 +- services/dask-sidecar/tests/unit/test_worker.py | 3 ++- 5 files changed, 6 insertions(+), 5 deletions(-) rename services/dask-sidecar/src/simcore_service_dask_sidecar/{rabbitmq_plugin.py => rabbitmq_worker_plugin.py} (99%) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py similarity index 99% rename from services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_plugin.py rename to services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py index 554988aa0b2f..4cd6304bcdba 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py @@ -24,7 +24,7 @@ class RabbitMQPlugin(distributed.WorkerPlugin): """Dask Worker Plugin for RabbitMQ integration""" - name = "rabbitmq_plugin" + name = "rabbitmq_worker_plugin" _main_thread_loop: AbstractEventLoop | None = None _client: RabbitMQClient | None = None _settings: RabbitSettings | None = None diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/dask.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/dask.py index 1b4b40c8d348..116f3a70adfe 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/dask.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/dask.py @@ -19,7 +19,7 @@ from models_library.rabbitmq_messages import LoggerRabbitMessage from servicelib.logging_utils import LogLevelInt, LogMessageStr, log_catch, log_context -from ..rabbitmq_plugin import get_rabbitmq_client +from ..rabbitmq_worker_plugin import get_rabbitmq_client _logger = logging.getLogger(__name__) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py index df6b4a482664..41b049e2c908 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py @@ -16,7 +16,7 @@ from ._meta import print_dask_sidecar_banner from .computational_sidecar.core import ComputationalSidecar -from .rabbitmq_plugin import RabbitMQPlugin +from .rabbitmq_worker_plugin import RabbitMQPlugin from .settings import ApplicationSettings from .task_life_cycle_worker_plugin import ( TaskLifecycleWorkerPlugin, diff --git a/services/dask-sidecar/tests/unit/test_rabbitmq_plugin.py b/services/dask-sidecar/tests/unit/test_rabbitmq_plugin.py index dd644d64fa06..6d2b56cf57ee 100644 --- a/services/dask-sidecar/tests/unit/test_rabbitmq_plugin.py +++ b/services/dask-sidecar/tests/unit/test_rabbitmq_plugin.py @@ -24,7 +24,7 @@ def test_rabbitmq_plugin_initializes(dask_client: distributed.Client): ... @pytest.fixture def erroring_rabbitmq_plugin(mocker: MockerFixture) -> mock.Mock: return mocker.patch( - "simcore_service_dask_sidecar.rabbitmq_plugin.RabbitMQPlugin", + "simcore_service_dask_sidecar.rabbitmq_worker_plugin.RabbitMQPlugin", autospec=True, side_effect=RuntimeError("Pytest: RabbitMQ plugin initialization failed"), ) diff --git a/services/dask-sidecar/tests/unit/test_worker.py b/services/dask-sidecar/tests/unit/test_worker.py index 235b61e3d15b..08811ef62946 100644 --- a/services/dask-sidecar/tests/unit/test_worker.py +++ b/services/dask-sidecar/tests/unit/test_worker.py @@ -113,7 +113,8 @@ def dask_subsystem_mock( ) # mock dask rabbitmq plugin mock_dask_rabbitmq_plugin = mocker.patch( - "simcore_service_dask_sidecar.rabbitmq_plugin.RabbitMQPlugin", autospec=True + "simcore_service_dask_sidecar.rabbitmq_worker_plugin.RabbitMQPlugin", + autospec=True, ) mock_rabbitmq_client = create_rabbitmq_client("pytest_dask_sidecar_logs_publisher") mock_dask_rabbitmq_plugin.get_client.return_value = mock_rabbitmq_client From 937143a88e296d086f7ad18cc6674b20225ba252 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 08:11:23 +0200 Subject: [PATCH 07/46] refactor --- .../task_life_cycle_scheduler_plugin.py | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py index 9b33ccd9e902..b5150c33001f 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py @@ -1,14 +1,37 @@ import logging -from typing import Any +from dataclasses import asdict, dataclass +from typing import Any, Final from dask.typing import Key from distributed import Scheduler, SchedulerPlugin from distributed.scheduler import TaskStateState +from models_library.projects_state import RunningState from servicelib.logging_utils import log_context _logger = logging.getLogger(__name__) +_TASK_LIFE_CYCLE_EVENT: Final[str] = "task-lifecycle-{key}" +_SCHEDULER_TASK_STATE_TO_RUNNING_STATE: Final[dict[TaskStateState, RunningState]] = {} + + +@dataclass +class TaskLifeCycleState: + key: Key + worker: str | None + state: RunningState + + @classmethod + def from_scheduler_task_state( + cls, key: Key, worker: str | None, task_state: TaskStateState + ) -> "TaskLifeCycleState": + return cls( + key=key, + worker=worker, + state=_SCHEDULER_TASK_STATE_TO_RUNNING_STATE[task_state], + ) + + class TaskLifecycleSchedulerPlugin(SchedulerPlugin): def __init__(self) -> None: with log_context( @@ -31,7 +54,7 @@ def transition( key: Key, start: TaskStateState, finish: TaskStateState, - *args: Any, + *args: Any, # noqa: ARG002 stimulus_id: str, **kwargs: Any, ): @@ -43,12 +66,10 @@ def transition( ): assert self.scheduler # nosec self.scheduler.log_event( - f"task-lifecycle-{key}", - { - "key": key, - "worker": kwargs.get("worker"), - "start": start, - "finish": finish, - "stimulus_id": stimulus_id, - }, + _TASK_LIFE_CYCLE_EVENT.format(key=key), + asdict( + TaskLifeCycleState.from_scheduler_task_state( + key, kwargs.get("worker"), finish + ) + ), ) From 4014d3daf1dbb5bb0052e9cf2facc6336d4b2568 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 08:25:38 +0200 Subject: [PATCH 08/46] ongoing --- .../src/dask_task_models_library/models.py | 59 ++++++++++++++++++- .../task_life_cycle_scheduler_plugin.py | 36 ++--------- 2 files changed, 64 insertions(+), 31 deletions(-) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/models.py b/packages/dask-task-models-library/src/dask_task_models_library/models.py index 2e0bbed7535b..f21f1d1299a3 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/models.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/models.py @@ -1,4 +1,61 @@ -from typing import TypeAlias +from dataclasses import asdict, dataclass +from typing import Any, Final, TypeAlias + +from dask.typing import Key +from distributed.scheduler import TaskStateState as SchedulerTaskState +from distributed.worker_state_machine import TaskStateState as WorkerTaskState +from models_library.projects_state import RunningState DaskJobID: TypeAlias = str DaskResources: TypeAlias = dict[str, int | float] + +TASK_LIFE_CYCLE_EVENT: Final[str] = "task-lifecycle-{key}" +_SCHEDULER_TASK_STATE_TO_RUNNING_STATE: Final[ + dict[SchedulerTaskState, RunningState] +] = { + "waiting": RunningState.PENDING, + "no-worker": RunningState.WAITING_FOR_RESOURCES, + "queued": RunningState.WAITING_FOR_RESOURCES, + "processing": RunningState.PENDING, + "memory": RunningState.SUCCESS, + "erred": RunningState.FAILED, + "forgotten": RunningState.UNKNOWN, +} + +_WORKER_TASK_STATE_TO_RUNNING_STATE: Final[dict[WorkerTaskState, RunningState]] = { + "cancelled": RunningState.UNKNOWN, + "constrained": RunningState.UNKNOWN, + "error": RunningState.UNKNOWN, + "executing": RunningState.UNKNOWN, + "fetch": RunningState.UNKNOWN, + "flight": RunningState.UNKNOWN, + "forgotten": RunningState.UNKNOWN, + "long-running": RunningState.UNKNOWN, + "memory": RunningState.UNKNOWN, + "missing": RunningState.UNKNOWN, + "ready": RunningState.UNKNOWN, + "released": RunningState.UNKNOWN, + "rescheduled": RunningState.UNKNOWN, + "resumed": RunningState.UNKNOWN, + "waiting": RunningState.UNKNOWN, +} + + +@dataclass +class TaskLifeCycleState: + key: Key + worker: str | None + state: RunningState + + @classmethod + def from_scheduler_task_state( + cls, key: Key, worker: str | None, task_state: SchedulerTaskState + ) -> "TaskLifeCycleState": + return cls( + key=key, + worker=worker, + state=_SCHEDULER_TASK_STATE_TO_RUNNING_STATE[task_state], + ) + + def model_dump(self) -> dict[str, Any]: + return asdict(self) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py index b5150c33001f..0779cddc3b47 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py @@ -1,37 +1,15 @@ import logging -from dataclasses import asdict, dataclass -from typing import Any, Final +from typing import Any from dask.typing import Key +from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState from distributed import Scheduler, SchedulerPlugin from distributed.scheduler import TaskStateState -from models_library.projects_state import RunningState from servicelib.logging_utils import log_context _logger = logging.getLogger(__name__) -_TASK_LIFE_CYCLE_EVENT: Final[str] = "task-lifecycle-{key}" -_SCHEDULER_TASK_STATE_TO_RUNNING_STATE: Final[dict[TaskStateState, RunningState]] = {} - - -@dataclass -class TaskLifeCycleState: - key: Key - worker: str | None - state: RunningState - - @classmethod - def from_scheduler_task_state( - cls, key: Key, worker: str | None, task_state: TaskStateState - ) -> "TaskLifeCycleState": - return cls( - key=key, - worker=worker, - state=_SCHEDULER_TASK_STATE_TO_RUNNING_STATE[task_state], - ) - - class TaskLifecycleSchedulerPlugin(SchedulerPlugin): def __init__(self) -> None: with log_context( @@ -66,10 +44,8 @@ def transition( ): assert self.scheduler # nosec self.scheduler.log_event( - _TASK_LIFE_CYCLE_EVENT.format(key=key), - asdict( - TaskLifeCycleState.from_scheduler_task_state( - key, kwargs.get("worker"), finish - ) - ), + TASK_LIFE_CYCLE_EVENT.format(key=key), + TaskLifeCycleState.from_scheduler_task_state( + key, kwargs.get("worker"), finish + ).model_dump(), ) From 9f5d9aabf4d3b962daae3abfe58b7236bf2f4373 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 15:57:16 +0200 Subject: [PATCH 09/46] sending events about task status --- .../src/dask_task_models_library/models.py | 70 +++++++++++-------- .../task_life_cycle_scheduler_plugin.py | 6 +- .../task_life_cycle_worker_plugin.py | 29 ++++---- .../dask-sidecar/tests/unit/test_scheduler.py | 11 ++- 4 files changed, 66 insertions(+), 50 deletions(-) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/models.py b/packages/dask-task-models-library/src/dask_task_models_library/models.py index f21f1d1299a3..42f961bd3b8c 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/models.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/models.py @@ -1,10 +1,10 @@ -from dataclasses import asdict, dataclass -from typing import Any, Final, TypeAlias +from typing import Final, Literal, TypeAlias from dask.typing import Key from distributed.scheduler import TaskStateState as SchedulerTaskState from distributed.worker_state_machine import TaskStateState as WorkerTaskState from models_library.projects_state import RunningState +from pydantic import BaseModel DaskJobID: TypeAlias = str DaskResources: TypeAlias = dict[str, int | float] @@ -13,37 +13,38 @@ _SCHEDULER_TASK_STATE_TO_RUNNING_STATE: Final[ dict[SchedulerTaskState, RunningState] ] = { - "waiting": RunningState.PENDING, - "no-worker": RunningState.WAITING_FOR_RESOURCES, - "queued": RunningState.WAITING_FOR_RESOURCES, - "processing": RunningState.PENDING, - "memory": RunningState.SUCCESS, - "erred": RunningState.FAILED, - "forgotten": RunningState.UNKNOWN, + "released": RunningState.NOT_STARTED, # Known but not actively computing or in memory + "waiting": RunningState.PENDING, # On track to be computed, waiting on dependencies to arrive in memory + "no-worker": RunningState.WAITING_FOR_RESOURCES, # Ready to be computed, but no appropriate worker exists (for example because of resource restrictions, or because no worker is connected at all). + "queued": RunningState.WAITING_FOR_RESOURCES, # Ready to be computed, but all workers are already full. + "processing": RunningState.PENDING, # All dependencies are available and the task is assigned to a worker for compute (the scheduler doesn’t know whether it’s in a worker queue or actively being computed). + "memory": RunningState.SUCCESS, # In memory on one or more workers + "erred": RunningState.FAILED, # Task computation, or one of its dependencies, has encountered an error + "forgotten": RunningState.UNKNOWN, # Task is no longer needed by any client or dependent task, so it disappears from the scheduler as well. As soon as a task reaches this state, it is immediately dereferenced from the scheduler. } _WORKER_TASK_STATE_TO_RUNNING_STATE: Final[dict[WorkerTaskState, RunningState]] = { - "cancelled": RunningState.UNKNOWN, - "constrained": RunningState.UNKNOWN, - "error": RunningState.UNKNOWN, - "executing": RunningState.UNKNOWN, - "fetch": RunningState.UNKNOWN, - "flight": RunningState.UNKNOWN, - "forgotten": RunningState.UNKNOWN, - "long-running": RunningState.UNKNOWN, - "memory": RunningState.UNKNOWN, - "missing": RunningState.UNKNOWN, - "ready": RunningState.UNKNOWN, - "released": RunningState.UNKNOWN, - "rescheduled": RunningState.UNKNOWN, - "resumed": RunningState.UNKNOWN, - "waiting": RunningState.UNKNOWN, + "cancelled": RunningState.ABORTED, # The scheduler asked to forget about this task, but it’s technically impossible at the moment. See Task cancellation. The task can be found in whatever collections it was in its previous state. + "constrained": RunningState.PENDING, # Like ready, but the user specified resource constraints for this task. The task can be found in the WorkerState.constrained queue. + "error": RunningState.FAILED, # Task execution failed + "executing": RunningState.STARTED, # The task is currently being computed on a thread. It can be found in the WorkerState.executing set and in the distributed.worker.Worker.active_threads dict. + "fetch": RunningState.PENDING, # This task is in memory on one or more peer workers, but not on this worker. Its data is queued to be transferred over the network, either because it’s a dependency of a task in waiting state, or because the Active Memory Manager requested it to be replicated here. The task can be found in the WorkerState.data_needed heap. + "flight": RunningState.PENDING, # The task data is currently being transferred over the network from another worker. The task can be found in the WorkerState.in_flight_tasks and WorkerState.in_flight_workers collections. + "forgotten": RunningState.UNKNOWN, # The scheduler asked this worker to forget about the task, and there are neither dependents nor dependencies on the same worker. + "long-running": RunningState.STARTED, # Like executing, but the user code called distributed.secede() so the task no longer counts towards the maximum number of concurrent tasks. It can be found in the WorkerState.long_running set and in the distributed.worker.Worker.active_threads dict. + "memory": RunningState.SUCCESS, # Task execution completed, or the task was successfully transferred from another worker, and is now held in either WorkerState.data or WorkerState.actors. + "missing": RunningState.PENDING, # Like fetch, but all peer workers that were listed by the scheduler are either unreachable or have responded they don’t actually have the task data. The worker will periodically ask the scheduler if it knows of additional replicas; when it does, the task will transition again to fetch. The task can be found in the WorkerState.missing_dep_flight set. + "ready": RunningState.PENDING, # The task is ready to be computed; all of its dependencies are in memory on the current worker and it’s waiting for an available thread. The task can be found in the WorkerState.ready heap. + "released": RunningState.PENDING, # Known but not actively computing or in memory. A task can stay in this state when the scheduler asked to forget it, but it has dependent tasks on the same worker. + "rescheduled": RunningState.PENDING, # The task just raised the Reschedule exception. This is a transitory state, which is not stored permanently. + "resumed": RunningState.PENDING, # The task was recovered from cancelled state. See Task cancellation. The task can be found in whatever collections it was in its previous state. + "waiting": RunningState.PENDING, # The scheduler has added the task to the worker queue. All of its dependencies are in memory somewhere on the cluster, but not all of them are in memory on the current worker, so they need to be fetched. } -@dataclass -class TaskLifeCycleState: - key: Key +class TaskLifeCycleState(BaseModel): + key: str + source: Literal["scheduler", "worker"] worker: str | None state: RunningState @@ -52,10 +53,19 @@ def from_scheduler_task_state( cls, key: Key, worker: str | None, task_state: SchedulerTaskState ) -> "TaskLifeCycleState": return cls( - key=key, + key=f"{key!r}", + source="scheduler", worker=worker, state=_SCHEDULER_TASK_STATE_TO_RUNNING_STATE[task_state], ) - def model_dump(self) -> dict[str, Any]: - return asdict(self) + @classmethod + def from_worker_task_state( + cls, key: Key, worker: str | None, task_state: WorkerTaskState + ) -> "TaskLifeCycleState": + return cls( + key=f"{key!r}", + source="worker", + worker=worker, + state=_WORKER_TASK_STATE_TO_RUNNING_STATE[task_state], + ) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py index 0779cddc3b47..82124ae84fca 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py @@ -36,16 +36,16 @@ def transition( stimulus_id: str, **kwargs: Any, ): - # Start state: one of released, waiting, processing, memory, error with log_context( _logger, logging.INFO, - f"Task {key} transition from {start} to {finish} due to {stimulus_id=}", + f"Task {key!r} transition from {start} to {finish} due to {stimulus_id=}", ): assert self.scheduler # nosec + self.scheduler.log_event( TASK_LIFE_CYCLE_EVENT.format(key=key), TaskLifeCycleState.from_scheduler_task_state( key, kwargs.get("worker"), finish - ).model_dump(), + ).model_dump(mode="json"), ) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py index d384c02cda58..9e5226c0b2f1 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py @@ -3,8 +3,10 @@ from typing import Any from dask.typing import Key -from distributed import Worker, WorkerPlugin -from distributed.scheduler import TaskStateState +from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState +from distributed import WorkerPlugin +from distributed.worker import Worker +from distributed.worker_state_machine import TaskStateState from servicelib.logging_utils import log_context _logger = logging.getLogger(__name__) @@ -17,7 +19,7 @@ def __init__(self) -> None: logging.INFO, "TaskLifecycleWorkerPlugin init", ): - self.worker = None + self._worker = None def setup(self, worker: Worker) -> Awaitable[None]: async def _() -> None: @@ -26,7 +28,8 @@ async def _() -> None: logging.INFO, "TaskLifecycleWorkerPlugin start", ): - self.worker = worker + assert worker # nosec + self._worker = worker return _() @@ -37,19 +40,15 @@ def transition( finish: TaskStateState, **kwargs: Any, ): - # Start state: one of released, waiting, processing, memory, error with log_context( _logger, logging.INFO, - f"Task {key} transition from {start} to {finish}", + f"Task {key!r} transition from {start} to {finish}", ): - assert self.worker # nosec - self.worker.log_event( - f"task-lifecycle-{key}", - { - "key": key, - "worker": kwargs.get("worker"), - "start": start, - "finish": finish, - }, + assert self._worker # nosec + self._worker.log_event( + TASK_LIFE_CYCLE_EVENT.format(key=key), + TaskLifeCycleState.from_worker_task_state( + key, kwargs.get("worker"), finish + ).model_dump(mode="json"), ) diff --git a/services/dask-sidecar/tests/unit/test_scheduler.py b/services/dask-sidecar/tests/unit/test_scheduler.py index 7f709cb5c370..0c28340c12cc 100644 --- a/services/dask-sidecar/tests/unit/test_scheduler.py +++ b/services/dask-sidecar/tests/unit/test_scheduler.py @@ -1,7 +1,13 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=no-member + import time import distributed import pytest +from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT pytest_simcore_core_services_selection = [ "rabbit", @@ -20,7 +26,8 @@ def _some_failing_task() -> None: future = dask_client.submit(_some_task) assert future.result(timeout=10) == 2 - events = dask_client.get_events(f"task-lifecycle-{future.key}") + + events = dask_client.get_events(TASK_LIFE_CYCLE_EVENT.format(key=future.key)) print("XXXX received events:") assert events assert isinstance(events, tuple) @@ -30,7 +37,7 @@ def _some_failing_task() -> None: future = dask_client.submit(_some_failing_task) with pytest.raises(RuntimeError): future.result(timeout=10) - events = dask_client.get_events(f"task-lifecycle-{future.key}") + events = dask_client.get_events(TASK_LIFE_CYCLE_EVENT.format(key=future.key)) print("XXXX received events:") assert events assert isinstance(events, tuple) From a451c391ac919ece5ee1a75b67cf0035380a38bf Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 16:02:37 +0200 Subject: [PATCH 10/46] check returned states --- .../dask-sidecar/tests/unit/test_scheduler.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/services/dask-sidecar/tests/unit/test_scheduler.py b/services/dask-sidecar/tests/unit/test_scheduler.py index 0c28340c12cc..889de0e5887a 100644 --- a/services/dask-sidecar/tests/unit/test_scheduler.py +++ b/services/dask-sidecar/tests/unit/test_scheduler.py @@ -7,7 +7,8 @@ import distributed import pytest -from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT +from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState +from models_library.projects_state import RunningState pytest_simcore_core_services_selection = [ "rabbit", @@ -28,18 +29,20 @@ def _some_failing_task() -> None: assert future.result(timeout=10) == 2 events = dask_client.get_events(TASK_LIFE_CYCLE_EVENT.format(key=future.key)) - print("XXXX received events:") - assert events assert isinstance(events, tuple) - for event in events: - print(f"\t{event}") + parsed_events = [TaskLifeCycleState.model_validate(event[1]) for event in events] + assert parsed_events[0].state is RunningState.PENDING + assert RunningState.STARTED in {event.state for event in parsed_events} + assert RunningState.FAILED not in {event.state for event in parsed_events} + assert parsed_events[-1].state is RunningState.SUCCESS future = dask_client.submit(_some_failing_task) with pytest.raises(RuntimeError): future.result(timeout=10) events = dask_client.get_events(TASK_LIFE_CYCLE_EVENT.format(key=future.key)) - print("XXXX received events:") - assert events - assert isinstance(events, tuple) - for event in events: - print(f"\t{event}") + parsed_events = [TaskLifeCycleState.model_validate(event[1]) for event in events] + assert parsed_events[0].state is RunningState.PENDING + assert RunningState.STARTED in {event.state for event in parsed_events} + assert RunningState.FAILED in {event.state for event in parsed_events} + assert RunningState.SUCCESS not in {event.state for event in parsed_events} + assert parsed_events[-1].state is RunningState.FAILED From a3a3cbfa69d9a4dfdc0a6194a8bec162adbda8eb Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 16:05:50 +0200 Subject: [PATCH 11/46] rename --- .../tests/unit/{test_scheduler.py => test_base_tasks.py} | 2 +- .../{test_worker.py => test_computational_sidecar_tasks.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename services/dask-sidecar/tests/unit/{test_scheduler.py => test_base_tasks.py} (96%) rename services/dask-sidecar/tests/unit/{test_worker.py => test_computational_sidecar_tasks.py} (99%) diff --git a/services/dask-sidecar/tests/unit/test_scheduler.py b/services/dask-sidecar/tests/unit/test_base_tasks.py similarity index 96% rename from services/dask-sidecar/tests/unit/test_scheduler.py rename to services/dask-sidecar/tests/unit/test_base_tasks.py index 889de0e5887a..37ff4a040f4d 100644 --- a/services/dask-sidecar/tests/unit/test_scheduler.py +++ b/services/dask-sidecar/tests/unit/test_base_tasks.py @@ -15,7 +15,7 @@ ] -def test_scheduler(dask_client: distributed.Client) -> None: +def test_task_state_lifecycle(dask_client: distributed.Client) -> None: def _some_task() -> int: time.sleep(1) return 2 diff --git a/services/dask-sidecar/tests/unit/test_worker.py b/services/dask-sidecar/tests/unit/test_computational_sidecar_tasks.py similarity index 99% rename from services/dask-sidecar/tests/unit/test_worker.py rename to services/dask-sidecar/tests/unit/test_computational_sidecar_tasks.py index 08811ef62946..a0756a0bc308 100644 --- a/services/dask-sidecar/tests/unit/test_worker.py +++ b/services/dask-sidecar/tests/unit/test_computational_sidecar_tasks.py @@ -506,7 +506,7 @@ async def subscribe_and_process(a_mock: mock.AsyncMock): ready_event.set() # Wait until the test is done - while not shutdown_event.is_set(): + while not shutdown_event.is_set(): # noqa: ASYNC110 await asyncio.sleep(0.1) # Cleanup From f46aff50f4a182377a78e83924b617ea103be04b Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 16:06:49 +0200 Subject: [PATCH 12/46] mypy --- .../task_life_cycle_scheduler_plugin.py | 2 +- .../task_life_cycle_worker_plugin.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py index 82124ae84fca..c0ac1ce7b1bf 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py @@ -25,7 +25,7 @@ async def start(self, scheduler: Scheduler) -> None: logging.INFO, "TaskLifecycleSchedulerPlugin start", ): - self.scheduler = scheduler + self.scheduler = scheduler # type: ignore[assignment] def transition( self, diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py index 9e5226c0b2f1..ba07fa5836f3 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py @@ -28,8 +28,7 @@ async def _() -> None: logging.INFO, "TaskLifecycleWorkerPlugin start", ): - assert worker # nosec - self._worker = worker + self._worker = worker # type: ignore[assignment] return _() From 38c0a7a19775a0f49f3eb13cd8e305d5d4718ff5 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 16:30:44 +0200 Subject: [PATCH 13/46] seems to work --- .../rabbitmq_worker_plugin.py | 2 +- .../tests/unit/test_base_tasks.py | 48 ----------- .../tests/unit/test_tasks_life_cycle.py | 80 +++++++++++++++++++ 3 files changed, 81 insertions(+), 49 deletions(-) delete mode 100644 services/dask-sidecar/tests/unit/test_base_tasks.py create mode 100644 services/dask-sidecar/tests/unit/test_tasks_life_cycle.py diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py index 4cd6304bcdba..616a564b3850 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py @@ -98,7 +98,7 @@ async def _() -> None: ) else: _logger.warning( - "RabbitMQ client plugin setup is not the main thread!" + "RabbitMQ client plugin setup is not the main thread! Beware! if in pytest it's ok." ) # Cancel the message processor task diff --git a/services/dask-sidecar/tests/unit/test_base_tasks.py b/services/dask-sidecar/tests/unit/test_base_tasks.py deleted file mode 100644 index 37ff4a040f4d..000000000000 --- a/services/dask-sidecar/tests/unit/test_base_tasks.py +++ /dev/null @@ -1,48 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable -# pylint: disable=no-member - -import time - -import distributed -import pytest -from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState -from models_library.projects_state import RunningState - -pytest_simcore_core_services_selection = [ - "rabbit", -] - - -def test_task_state_lifecycle(dask_client: distributed.Client) -> None: - def _some_task() -> int: - time.sleep(1) - return 2 - - def _some_failing_task() -> None: - time.sleep(1) - msg = "Some error" - raise RuntimeError(msg) - - future = dask_client.submit(_some_task) - assert future.result(timeout=10) == 2 - - events = dask_client.get_events(TASK_LIFE_CYCLE_EVENT.format(key=future.key)) - assert isinstance(events, tuple) - parsed_events = [TaskLifeCycleState.model_validate(event[1]) for event in events] - assert parsed_events[0].state is RunningState.PENDING - assert RunningState.STARTED in {event.state for event in parsed_events} - assert RunningState.FAILED not in {event.state for event in parsed_events} - assert parsed_events[-1].state is RunningState.SUCCESS - - future = dask_client.submit(_some_failing_task) - with pytest.raises(RuntimeError): - future.result(timeout=10) - events = dask_client.get_events(TASK_LIFE_CYCLE_EVENT.format(key=future.key)) - parsed_events = [TaskLifeCycleState.model_validate(event[1]) for event in events] - assert parsed_events[0].state is RunningState.PENDING - assert RunningState.STARTED in {event.state for event in parsed_events} - assert RunningState.FAILED in {event.state for event in parsed_events} - assert RunningState.SUCCESS not in {event.state for event in parsed_events} - assert parsed_events[-1].state is RunningState.FAILED diff --git a/services/dask-sidecar/tests/unit/test_tasks_life_cycle.py b/services/dask-sidecar/tests/unit/test_tasks_life_cycle.py new file mode 100644 index 000000000000..5e894f36447f --- /dev/null +++ b/services/dask-sidecar/tests/unit/test_tasks_life_cycle.py @@ -0,0 +1,80 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=no-member + +import time + +import distributed +import pytest +from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState +from models_library.projects_state import RunningState +from tenacity import Retrying, stop_after_delay, wait_fixed + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +def test_task_state_lifecycle(local_cluster: distributed.LocalCluster) -> None: + def _some_task() -> int: + time.sleep(1) + return 2 + + def _some_failing_task() -> None: + time.sleep(1) + msg = "Some error" + raise RuntimeError(msg) + + local_cluster.scale(0) + for attempt in Retrying( + stop=stop_after_delay(10), wait=wait_fixed(1), reraise=True + ): + with attempt: + assert len(local_cluster.workers) == 0 + with distributed.Client(local_cluster) as dask_client: + # submit the task and wait until it goes into WAITING_FOR_RESOURCES + future = dask_client.submit(_some_task, resources={"CPU": 1}) + for attempt in Retrying( + stop=stop_after_delay(10), wait=wait_fixed(1), reraise=True + ): + with attempt: + events = dask_client.get_events( + TASK_LIFE_CYCLE_EVENT.format(key=future.key) + ) + assert isinstance(events, tuple) + assert len(events) >= 2 + parsed_events = [ + TaskLifeCycleState.model_validate(event[1]) for event in events + ] + assert parsed_events[0].state is RunningState.PENDING + assert parsed_events[-1].state is RunningState.WAITING_FOR_RESOURCES + + # now add a worker and wait for it to take the task + local_cluster.scale(1) + + # we basically wait for the tasks to finish + assert future.result(timeout=15) == 2 + + events = dask_client.get_events(TASK_LIFE_CYCLE_EVENT.format(key=future.key)) + assert isinstance(events, tuple) + parsed_events = [ + TaskLifeCycleState.model_validate(event[1]) for event in events + ] + assert parsed_events[0].state is RunningState.PENDING + assert RunningState.STARTED in {event.state for event in parsed_events} + assert RunningState.FAILED not in {event.state for event in parsed_events} + assert parsed_events[-1].state is RunningState.SUCCESS + + future = dask_client.submit(_some_failing_task) + with pytest.raises(RuntimeError): + future.result(timeout=10) + events = dask_client.get_events(TASK_LIFE_CYCLE_EVENT.format(key=future.key)) + parsed_events = [ + TaskLifeCycleState.model_validate(event[1]) for event in events + ] + assert parsed_events[0].state is RunningState.PENDING + assert RunningState.STARTED in {event.state for event in parsed_events} + assert RunningState.FAILED in {event.state for event in parsed_events} + assert RunningState.SUCCESS not in {event.state for event in parsed_events} + assert parsed_events[-1].state is RunningState.FAILED From ced886095a6473a6dc9b847b1c440fd1e93829e2 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 16:53:28 +0200 Subject: [PATCH 14/46] added get_task_status2 --- .../modules/dask_client.py | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index a5dcfc4fd7bc..46e3f6667bfa 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -8,9 +8,10 @@ """ +import asyncio import logging import traceback -from collections.abc import Callable +from collections.abc import Callable, Iterable from copy import deepcopy from dataclasses import dataclass from http.client import HTTPException @@ -38,7 +39,12 @@ TaskOwner, ) from dask_task_models_library.container_tasks.utils import generate_dask_job_id -from dask_task_models_library.models import DaskJobID, DaskResources +from dask_task_models_library.models import ( + TASK_LIFE_CYCLE_EVENT, + DaskJobID, + DaskResources, + TaskLifeCycleState, +) from dask_task_models_library.resource_constraints import ( create_ec2_resource_constraint_key, ) @@ -48,6 +54,7 @@ from models_library.clusters import ClusterAuthentication, ClusterTypeInModel from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID +from models_library.projects_state import RunningState from models_library.resource_tracker import HardwareInfo from models_library.services import ServiceRunID from models_library.users import UserID @@ -77,6 +84,7 @@ from ..utils.dask_client_utils import ( DaskSubSystem, TaskHandlers, + UnixTimestamp, connect_to_dask_scheduler, ) from .db import get_db_engine @@ -419,6 +427,57 @@ async def send_computation_tasks( return list_of_node_id_to_job_id + async def get_tasks_status2(self, job_ids: Iterable[str]) -> list[RunningState]: + dask_utils.check_scheduler_is_still_the_same( + self.backend.scheduler_id, self.backend.client + ) + dask_utils.check_communication_with_scheduler_is_open(self.backend.client) + dask_utils.check_scheduler_status(self.backend.client) + + async def _get_job_id_status(job_id: str) -> RunningState: + # TODO: maybe we should define an event just for that, instead of multiple calls + dask_events: tuple[tuple[UnixTimestamp, str], ...] = ( + await self.backend.client.get_events( + TASK_LIFE_CYCLE_EVENT.format(key=job_id) + ) + ) + if not dask_events: + return RunningState.UNKNOWN + # we are interested in the last event + parsed_event = TaskLifeCycleState.model_validate(dask_events[-1][1]) + + if parsed_event.state == RunningState.FAILED: + try: + # find out if this was a cancellation + var = distributed.Variable(job_id, client=self.backend.client) + future: distributed.Future = await var.get( + timeout=_DASK_DEFAULT_TIMEOUT_S + ) + exception = await future.exception(timeout=_DASK_DEFAULT_TIMEOUT_S) + assert isinstance(exception, Exception) # nosec + + if isinstance(exception, TaskCancelledError): + return RunningState.ABORTED + assert exception # nosec + _logger.warning( + "Task %s completed in error:\n%s\nTrace:\n%s", + job_id, + exception, + "".join(traceback.format_exception(exception)), + ) + return RunningState.FAILED + except TimeoutError: + _logger.warning( + "Task %s could not be retrieved from dask-scheduler, it is lost\n" + "TIP:If the task was unpublished this can happen, or if the dask-scheduler was restarted.", + job_id, + ) + return RunningState.UNKNOWN + + return parsed_event.state + + return await asyncio.gather(*(_get_job_id_status(job_id) for job_id in job_ids)) + async def get_tasks_status(self, job_ids: list[str]) -> list[DaskClientTaskState]: dask_utils.check_scheduler_is_still_the_same( self.backend.scheduler_id, self.backend.client From 93dc096e22d9c05dfa657df9396c8d9e8db986f3 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 16 May 2025 17:33:27 +0200 Subject: [PATCH 15/46] ongoing removal of old stuff --- .../modules/comp_scheduler/_scheduler_dask.py | 99 +++++++++---------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py index ab31bcd13679..8dc940083379 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py @@ -38,14 +38,12 @@ ) from ...utils.dask_client_utils import TaskHandlers, UnixTimestamp from ...utils.rabbitmq import ( - publish_service_progress, publish_service_resource_tracking_stopped, publish_service_stopped_metrics, ) from ..clusters_keeper import get_or_create_on_demand_cluster from ..dask_client import DaskClient, PublishedComputationTask from ..dask_clients_pool import DaskClientsPool -from ..db.repositories.comp_runs import CompRunsRepository from ..db.repositories.comp_tasks import CompTasksRepository from ._scheduler_base import BaseCompScheduler @@ -159,27 +157,28 @@ async def _get_tasks_status( use_on_demand_clusters=comp_run.use_on_demand_clusters, run_metadata=comp_run.metadata, ) as client: - tasks_statuses = await client.get_tasks_status( - [f"{t.job_id}" for t in tasks] - ) - # process dask states - running_states: list[RunningState] = [] - for dask_task_state, task in zip(tasks_statuses, tasks, strict=True): - if dask_task_state is DaskClientTaskState.PENDING_OR_STARTED: - running_states += [ - ( - RunningState.STARTED - if task.progress is not None - else RunningState.PENDING - ) - ] - else: - running_states += [ - _DASK_CLIENT_TASK_STATE_TO_RUNNING_STATE_MAP.get( - dask_task_state, RunningState.UNKNOWN - ) - ] - return running_states + return await client.get_tasks_status2([f"{t.job_id}" for t in tasks]) + # tasks_statuses = await client.get_tasks_status( + # [f"{t.job_id}" for t in tasks] + # ) + # # process dask states + # running_states: list[RunningState] = [] + # for dask_task_state, task in zip(tasks_statuses, tasks, strict=True): + # if dask_task_state is DaskClientTaskState.PENDING_OR_STARTED: + # running_states += [ + # ( + # RunningState.STARTED + # if task.progress is not None + # else RunningState.PENDING + # ) + # ] + # else: + # running_states += [ + # _DASK_CLIENT_TASK_STATE_TO_RUNNING_STATE_MAP.get( + # dask_task_state, RunningState.UNKNOWN + # ) + # ] + # return running_states except ComputationalBackendOnDemandNotReadyError: _logger.info("The on demand computational backend is not ready yet...") @@ -351,30 +350,30 @@ async def _task_progress_change_handler( with log_catch(_logger, reraise=False): task_progress_event = TaskProgressEvent.model_validate_json(event[1]) _logger.debug("received task progress update: %s", task_progress_event) - user_id = task_progress_event.task_owner.user_id - project_id = task_progress_event.task_owner.project_id - node_id = task_progress_event.task_owner.node_id - comp_tasks_repo = CompTasksRepository(self.db_engine) - task = await comp_tasks_repo.get_task(project_id, node_id) - if task.progress is None: - task.state = RunningState.STARTED - task.progress = task_progress_event.progress - run = await CompRunsRepository(self.db_engine).get(user_id, project_id) - await self._process_started_tasks( - [task], - user_id=user_id, - project_id=project_id, - iteration=run.iteration, - run_metadata=run.metadata, - ) - else: - await comp_tasks_repo.update_project_task_progress( - project_id, node_id, task_progress_event.progress - ) - await publish_service_progress( - self.rabbitmq_client, - user_id=user_id, - project_id=project_id, - node_id=node_id, - progress=task_progress_event.progress, - ) + # user_id = task_progress_event.task_owner.user_id + # project_id = task_progress_event.task_owner.project_id + # node_id = task_progress_event.task_owner.node_id + # comp_tasks_repo = CompTasksRepository(self.db_engine) + # task = await comp_tasks_repo.get_task(project_id, node_id) + # if task.progress is None: + # task.state = RunningState.STARTED + # task.progress = task_progress_event.progress + # run = await CompRunsRepository(self.db_engine).get(user_id, project_id) + # await self._process_started_tasks( + # [task], + # user_id=user_id, + # project_id=project_id, + # iteration=run.iteration, + # run_metadata=run.metadata, + # ) + # else: + # await comp_tasks_repo.update_project_task_progress( + # project_id, node_id, task_progress_event.progress + # ) + # await publish_service_progress( + # self.rabbitmq_client, + # user_id=user_id, + # project_id=project_id, + # node_id=node_id, + # progress=task_progress_event.progress, + # ) From c1ea3f837f6a576def3ff877d9dc9c49270e4d87 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 08:09:33 +0200 Subject: [PATCH 16/46] ongoing --- services/dask-sidecar/tests/unit/test_tasks_life_cycle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/dask-sidecar/tests/unit/test_tasks_life_cycle.py b/services/dask-sidecar/tests/unit/test_tasks_life_cycle.py index 5e894f36447f..2b8b077cbe08 100644 --- a/services/dask-sidecar/tests/unit/test_tasks_life_cycle.py +++ b/services/dask-sidecar/tests/unit/test_tasks_life_cycle.py @@ -4,6 +4,7 @@ # pylint: disable=no-member import time +from collections.abc import Iterable import distributed import pytest @@ -70,6 +71,7 @@ def _some_failing_task() -> None: with pytest.raises(RuntimeError): future.result(timeout=10) events = dask_client.get_events(TASK_LIFE_CYCLE_EVENT.format(key=future.key)) + assert isinstance(events, Iterable) parsed_events = [ TaskLifeCycleState.model_validate(event[1]) for event in events ] From b63f4fa5a6fe2d0e4eb6dd63839b0a7808994af8 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 08:33:26 +0200 Subject: [PATCH 17/46] removed daskstate and upgrade test using running state --- .../models/dask_subsystem.py | 13 ---- .../modules/comp_scheduler/_scheduler_dask.py | 2 +- .../modules/dask_client.py | 72 +------------------ .../tests/unit/test_modules_dask_client.py | 45 ++++++------ 4 files changed, 27 insertions(+), 105 deletions(-) delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/dask_subsystem.py diff --git a/services/director-v2/src/simcore_service_director_v2/models/dask_subsystem.py b/services/director-v2/src/simcore_service_director_v2/models/dask_subsystem.py deleted file mode 100644 index 34a270cba88c..000000000000 --- a/services/director-v2/src/simcore_service_director_v2/models/dask_subsystem.py +++ /dev/null @@ -1,13 +0,0 @@ -from enum import Enum - - -# NOTE: mypy fails with src/simcore_service_director_v2/modules/dask_client.py:101:5: error: Dict entry 0 has incompatible type "str": "auto"; expected "Any": "DaskClientTaskState" [dict-item] -# when using StrAutoEnum -class DaskClientTaskState(str, Enum): - PENDING = "PENDING" - NO_WORKER = "NO_WORKER" - PENDING_OR_STARTED = "PENDING_OR_STARTED" - LOST = "LOST" - ERRED = "ERRED" - ABORTED = "ABORTED" - SUCCESS = "SUCCESS" diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py index 8dc940083379..646512e21be0 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py @@ -157,7 +157,7 @@ async def _get_tasks_status( use_on_demand_clusters=comp_run.use_on_demand_clusters, run_metadata=comp_run.metadata, ) as client: - return await client.get_tasks_status2([f"{t.job_id}" for t in tasks]) + return await client.get_tasks_status([f"{t.job_id}" for t in tasks]) # tasks_statuses = await client.get_tasks_status( # [f"{t.job_id}" for t in tasks] # ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index 46e3f6667bfa..c6c5a87dc121 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -17,7 +17,6 @@ from http.client import HTTPException from typing import Any, Final, cast -import dask.typing import distributed from aiohttp import ClientResponseError from common_library.json_serialization import json_dumps @@ -427,7 +426,7 @@ async def send_computation_tasks( return list_of_node_id_to_job_id - async def get_tasks_status2(self, job_ids: Iterable[str]) -> list[RunningState]: + async def get_tasks_status(self, job_ids: Iterable[str]) -> list[RunningState]: dask_utils.check_scheduler_is_still_the_same( self.backend.scheduler_id, self.backend.client ) @@ -436,6 +435,7 @@ async def get_tasks_status2(self, job_ids: Iterable[str]) -> list[RunningState]: async def _get_job_id_status(job_id: str) -> RunningState: # TODO: maybe we should define an event just for that, instead of multiple calls + # but the max length by default is 1000. We should test it dask_events: tuple[tuple[UnixTimestamp, str], ...] = ( await self.backend.client.get_events( TASK_LIFE_CYCLE_EVENT.format(key=job_id) @@ -478,74 +478,6 @@ async def _get_job_id_status(job_id: str) -> RunningState: return await asyncio.gather(*(_get_job_id_status(job_id) for job_id in job_ids)) - async def get_tasks_status(self, job_ids: list[str]) -> list[DaskClientTaskState]: - dask_utils.check_scheduler_is_still_the_same( - self.backend.scheduler_id, self.backend.client - ) - dask_utils.check_communication_with_scheduler_is_open(self.backend.client) - dask_utils.check_scheduler_status(self.backend.client) - - # try to get the task from the scheduler - def _get_pipeline_statuses( - dask_scheduler: distributed.Scheduler, - ) -> dict[dask.typing.Key, DaskSchedulerTaskState | None]: - statuses: dict[dask.typing.Key, DaskSchedulerTaskState | None] = ( - dask_scheduler.get_task_status(keys=job_ids) - ) - return statuses - - task_statuses: dict[dask.typing.Key, DaskSchedulerTaskState | None] = ( - await self.backend.client.run_on_scheduler(_get_pipeline_statuses) - ) - assert isinstance(task_statuses, dict) # nosec - - _logger.debug("found dask task statuses: %s", f"{task_statuses=}") - - running_states: list[DaskClientTaskState] = [] - for job_id in job_ids: - dask_status = cast( - DaskSchedulerTaskState | None, task_statuses.get(job_id, "lost") - ) - if dask_status == "erred": - try: - # find out if this was a cancellation - var = distributed.Variable(job_id, client=self.backend.client) - future: distributed.Future = await var.get( - timeout=_DASK_DEFAULT_TIMEOUT_S - ) - exception = await future.exception(timeout=_DASK_DEFAULT_TIMEOUT_S) - assert isinstance(exception, Exception) # nosec - - if isinstance(exception, TaskCancelledError): - running_states.append(DaskClientTaskState.ABORTED) - else: - assert exception # nosec - _logger.warning( - "Task %s completed in error:\n%s\nTrace:\n%s", - job_id, - exception, - "".join(traceback.format_exception(exception)), - ) - running_states.append(DaskClientTaskState.ERRED) - except TimeoutError: - _logger.warning( - "Task %s could not be retrieved from dask-scheduler, it is lost\n" - "TIP:If the task was unpublished this can happen, or if the dask-scheduler was restarted.", - job_id, - ) - running_states.append(DaskClientTaskState.LOST) - - elif dask_status is None: - running_states.append(DaskClientTaskState.LOST) - else: - running_states.append( - _DASK_TASK_STATUS_DASK_CLIENT_TASK_STATE_MAP.get( - dask_status, DaskClientTaskState.LOST - ) - ) - - return running_states - async def abort_computation_task(self, job_id: str) -> None: # Dask future may be cancelled, but only a future that was not already taken by # a sidecar can be cancelled that way. diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index 370cc967190d..534dc034661b 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -43,6 +43,7 @@ from models_library.docker import to_simcore_runtime_docker_label_key from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID +from models_library.projects_state import RunningState from models_library.resource_tracker import HardwareInfo from models_library.services_types import ServiceRunID from models_library.users import UserID @@ -60,7 +61,6 @@ ) from simcore_service_director_v2.models.comp_runs import RunMetadataDict from simcore_service_director_v2.models.comp_tasks import Image -from simcore_service_director_v2.models.dask_subsystem import DaskClientTaskState from simcore_service_director_v2.modules.dask_client import DaskClient, TaskHandlers from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -90,7 +90,7 @@ async def _assert_wait_for_cb_call(mocked_fct, timeout: int | None = None): async def _assert_wait_for_task_status( job_id: str, dask_client: DaskClient, - expected_status: DaskClientTaskState, + expected_status: RunningState, timeout: int | None = None, # noqa: ASYNC109 ): async for attempt in AsyncRetrying( @@ -105,11 +105,11 @@ async def _assert_wait_for_task_status( f"Attempt={attempt.retry_state.attempt_number}" ) got = (await dask_client.get_tasks_status([job_id]))[0] - assert isinstance(got, DaskClientTaskState) + assert isinstance(got, RunningState) print(f"{got=} vs {expected_status=}") - if got is DaskClientTaskState.ERRED and expected_status not in [ - DaskClientTaskState.ERRED, - DaskClientTaskState.LOST, + if got is RunningState.FAILED and expected_status not in [ + RunningState.FAILED, + RunningState.UNKNOWN, ]: try: # we can fail fast here @@ -516,7 +516,7 @@ def fake_sidecar_fct( await _assert_wait_for_task_status( published_computation_task.job_id, dask_client, - expected_status=DaskClientTaskState.PENDING_OR_STARTED, + expected_status=RunningState.STARTED, ) # using the event we let the remote fct continue @@ -530,7 +530,7 @@ def fake_sidecar_fct( await _assert_wait_for_task_status( published_computation_task.job_id, dask_client, - expected_status=DaskClientTaskState.SUCCESS, + expected_status=RunningState.SUCCESS, ) # check the results @@ -544,7 +544,7 @@ def fake_sidecar_fct( await _assert_wait_for_task_status( published_computation_task.job_id, dask_client, - expected_status=DaskClientTaskState.LOST, + expected_status=RunningState.UNKNOWN, timeout=60, ) @@ -609,7 +609,7 @@ def fake_sidecar_fct( await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, - expected_status=DaskClientTaskState.SUCCESS, + expected_status=RunningState.SUCCESS, ) assert published_computation_task[0].node_id in image_params.fake_tasks # creating a new future shows that it is not done???? @@ -704,7 +704,7 @@ def fake_remote_fct( await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, - DaskClientTaskState.PENDING_OR_STARTED, + RunningState.STARTED, ) # we wait to be sure the remote fct is started @@ -721,7 +721,7 @@ def fake_remote_fct( await _assert_wait_for_cb_call(mocked_user_completed_cb) await _assert_wait_for_task_status( - published_computation_task[0].job_id, dask_client, DaskClientTaskState.ABORTED + published_computation_task[0].job_id, dask_client, RunningState.ABORTED ) # getting the results should throw the cancellation error @@ -732,9 +732,12 @@ def fake_remote_fct( await dask_client.release_task_result(published_computation_task[0].job_id) # NOTE: this change of status takes a very long time to happen and is not relied upon so we skip it since it # makes the test fail a lot for no gain (it's kept here in case it ever becomes an issue) - # await _assert_wait_for_task_status( - # job_id, dask_client, RunningState.UNKNOWN, timeout=120 - # ) + await _assert_wait_for_task_status( + published_computation_task[0].job_id, + dask_client, + RunningState.UNKNOWN, + timeout=120, + ) async def test_failed_task_returns_exceptions( @@ -784,7 +787,7 @@ def fake_failing_sidecar_fct( await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, - expected_status=DaskClientTaskState.ERRED, + expected_status=RunningState.FAILED, ) with pytest.raises( ValueError, @@ -1047,7 +1050,7 @@ def fake_remote_fct( await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, - DaskClientTaskState.PENDING_OR_STARTED, + RunningState.STARTED, ) # let the remote fct run through now @@ -1057,7 +1060,7 @@ def fake_remote_fct( await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, - DaskClientTaskState.ERRED if fail_remote_fct else DaskClientTaskState.SUCCESS, + RunningState.FAILED if fail_remote_fct else RunningState.SUCCESS, ) # release the task results await dask_client.release_task_result(published_computation_task[0].job_id) @@ -1069,7 +1072,7 @@ def fake_remote_fct( await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, - DaskClientTaskState.LOST, + RunningState.UNKNOWN, timeout=60, ) @@ -1209,7 +1212,7 @@ def fake_sidecar_fct( await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, - expected_status=DaskClientTaskState.PENDING_OR_STARTED, + expected_status=RunningState.STARTED, ) # check we have one worker using the resources @@ -1240,7 +1243,7 @@ def fake_sidecar_fct( await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, - expected_status=DaskClientTaskState.SUCCESS, + expected_status=RunningState.SUCCESS, ) # check the resources are released From 75768fa0103c88e435a4f8cea021fd4035ea17b4 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 08:39:19 +0200 Subject: [PATCH 18/46] release constraint --- services/director-v2/requirements/_test.in | 6 +----- services/director-v2/requirements/_test.txt | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/services/director-v2/requirements/_test.in b/services/director-v2/requirements/_test.in index 3633a09b7044..2fb831189bae 100644 --- a/services/director-v2/requirements/_test.in +++ b/services/director-v2/requirements/_test.in @@ -19,11 +19,7 @@ docker Faker flaky pytest -# ---- -# Overcomes "Known issues" in https://github.com/pytest-dev/pytest-asyncio/releases/tag/v0.23.8 -# IMPORTANT: This constraint can be removed when `test_pytest_asyncio_known_issue` passes with the new update of pytest-asyncio -pytest-asyncio<0.23 -# ---- +pytest-asyncio pytest-cov pytest-docker pytest-icdiff diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index 08c3214f5da2..f28a0daa1544 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -232,7 +232,7 @@ pytest==8.3.5 # pytest-icdiff # pytest-mock # pytest-xdist -pytest-asyncio==0.21.2 +pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in From f6ea7c556b4a04b43b5b63bb4ccbb3f56a01b2e9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 08:52:56 +0200 Subject: [PATCH 19/46] added dependency to dask-sidecar --- services/director-v2/requirements/_test.in | 2 + services/director-v2/requirements/_test.txt | 593 +++++++++++++++++++- 2 files changed, 593 insertions(+), 2 deletions(-) diff --git a/services/director-v2/requirements/_test.in b/services/director-v2/requirements/_test.in index 2fb831189bae..8e2bf22b23bb 100644 --- a/services/director-v2/requirements/_test.in +++ b/services/director-v2/requirements/_test.in @@ -9,6 +9,8 @@ # --constraint _base.txt +--requirement ../../../services/dask-sidecar/requirements/_base.in # getting access to plugins + aio_pika aioboto3 alembic # migration due to pytest_simcore.postgres_service2 diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index f28a0daa1544..2bc9d5bb7d99 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -1,14 +1,32 @@ aio-pika==9.5.5 # via # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_test.in aioboto3==14.3.0 # via -r requirements/_test.in aiobotocore==2.22.0 - # via aioboto3 + # via + # aioboto3 + # s3fs +aiocache==0.12.3 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in +aiodebug==2.3.0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in +aiodocker==0.24.0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/_base.in aiofiles==24.1.0 # via # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/_base.in # aioboto3 aiohappyeyeballs==2.5.0 # via @@ -17,8 +35,29 @@ aiohappyeyeballs==2.5.0 aiohttp==3.11.18 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aiobotocore + # aiodocker + # fsspec + # s3fs aioitertools==0.12.0 # via aiobotocore aiormq==6.8.1 @@ -33,10 +72,23 @@ alembic==1.15.1 # via # -c requirements/_base.txt # -r requirements/_test.in +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic anyio==4.8.0 # via # -c requirements/_base.txt + # fast-depends + # faststream # httpx +arrow==1.3.0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in asgi-lifespan==2.1.0 # via -r requirements/_test.in async-asgi-testclient==1.4.11 @@ -45,7 +97,13 @@ attrs==25.1.0 # via # -c requirements/_base.txt # aiohttp + # jsonschema # pytest-docker + # referencing +blosc==1.11.2 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/_base.in bokeh==3.6.3 # via dask boto3==1.37.3 @@ -58,6 +116,24 @@ botocore==1.37.3 certifi==2025.1.31 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # httpcore # httpx @@ -71,6 +147,7 @@ click==8.1.7 # -c requirements/_base.txt # dask # distributed + # typer cloudpickle==3.1.0 # via # -c requirements/_base.txt @@ -82,15 +159,33 @@ coverage==7.6.12 # via pytest-cov dask==2024.12.0 # via + # -c requirements/../../../services/dask-sidecar/requirements/constraints.txt # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/_base.in # -r requirements/_test.in # distributed +deprecated==1.2.18 + # via + # -c requirements/_base.txt + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-semantic-conventions distributed==2024.12.0 # via # -c requirements/_base.txt # dask +dnspython==2.7.0 + # via + # -c requirements/_base.txt + # email-validator docker==7.1.0 # via -r requirements/_test.in +email-validator==2.2.0 + # via + # -c requirements/_base.txt + # pydantic exceptiongroup==1.2.2 # via # -c requirements/_base.txt @@ -99,6 +194,14 @@ execnet==2.1.1 # via pytest-xdist faker==36.2.2 # via -r requirements/_test.in +fast-depends==2.4.12 + # via + # -c requirements/_base.txt + # faststream +faststream==0.5.35 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in flaky==3.8.1 # via -r requirements/_test.in frozenlist==1.5.0 @@ -109,11 +212,22 @@ frozenlist==1.5.0 fsspec==2024.10.0 # via # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/_base.in # dask + # s3fs +googleapis-common-protos==1.69.1 + # via + # -c requirements/_base.txt + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http greenlet==3.1.1 # via # -c requirements/_base.txt # sqlalchemy +grpcio==1.70.0 + # via + # -c requirements/_base.txt + # opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # -c requirements/_base.txt @@ -125,6 +239,24 @@ httpcore==1.0.7 httpx==0.28.1 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # respx icdiff==2.0.7 @@ -133,6 +265,7 @@ idna==3.10 # via # -c requirements/_base.txt # anyio + # email-validator # httpx # requests # yarl @@ -140,11 +273,30 @@ importlib-metadata==8.5.0 # via # -c requirements/_base.txt # dask + # opentelemetry-api iniconfig==2.0.0 # via pytest jinja2==3.1.4 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # bokeh # dask @@ -154,21 +306,61 @@ jmespath==1.0.1 # aiobotocore # boto3 # botocore +jsonschema==4.23.0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +jsonschema-specifications==2024.10.1 + # via + # -c requirements/_base.txt + # jsonschema locket==1.0.0 # via # -c requirements/_base.txt # distributed # partd +lz4==4.3.3 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/_base.in mako==1.3.9 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # alembic +markdown-it-py==3.0.0 + # via + # -c requirements/_base.txt + # rich markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 # mako +mdurl==0.1.2 + # via + # -c requirements/_base.txt + # markdown-it-py msgpack==1.1.0 # via # -c requirements/_base.txt @@ -191,12 +383,117 @@ numpy==2.1.3 # contourpy # pandas # types-networkx +opentelemetry-api==1.30.0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-instrumentation-logging + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-otlp==1.30.0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-exporter-otlp-proto-common==1.30.0 + # via + # -c requirements/_base.txt + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.30.0 + # via + # -c requirements/_base.txt + # opentelemetry-exporter-otlp +opentelemetry-exporter-otlp-proto-http==1.30.0 + # via + # -c requirements/_base.txt + # opentelemetry-exporter-otlp +opentelemetry-instrumentation==0.51b0 + # via + # -c requirements/_base.txt + # opentelemetry-instrumentation-logging + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-logging==0.51b0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-redis==0.51b0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.51b0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-proto==1.30.0 + # via + # -c requirements/_base.txt + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.30.0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-semantic-conventions==0.51b0 + # via + # -c requirements/_base.txt + # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-sdk +opentelemetry-util-http==0.51b0 + # via + # -c requirements/_base.txt + # opentelemetry-instrumentation-requests +orjson==3.10.15 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.2 # via # -c requirements/_base.txt # bokeh # dask # distributed + # opentelemetry-instrumentation # pytest pamqp==3.3.0 # via @@ -214,15 +511,127 @@ pluggy==1.5.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff +prometheus-client==0.21.1 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/_base.in propcache==0.3.0 # via # -c requirements/_base.txt # aiohttp # yarl +protobuf==5.29.3 + # via + # -c requirements/_base.txt + # googleapis-common-protos + # opentelemetry-proto psutil==6.1.0 # via # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in # distributed +pycryptodome==3.21.0 + # via + # -c requirements/_base.txt + # stream-zip +pydantic==2.10.6 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/_base.in + # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.27.2 + # via + # -c requirements/_base.txt + # pydantic +pydantic-extra-types==2.10.2 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.7.0 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/_base.in +pygments==2.19.1 + # via + # -c requirements/_base.txt + # rich +pyinstrument==5.0.1 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in pytest==8.3.5 # via # -r requirements/_test.in @@ -235,6 +644,24 @@ pytest==8.3.5 pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -r requirements/_test.in pytest-cov==6.0.0 # via -r requirements/_test.in @@ -252,26 +679,118 @@ python-dateutil==2.9.0.post0 # via # -c requirements/_base.txt # aiobotocore + # arrow # botocore # pandas +python-dotenv==1.0.1 + # via + # -c requirements/_base.txt + # pydantic-settings pytz==2025.1 # via pandas pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in # bokeh # dask # distributed +redis==5.2.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in +referencing==0.35.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # jsonschema + # jsonschema-specifications +repro-zipfile==0.4.0 + # via -r requirements/../../../services/dask-sidecar/requirements/_base.in requests==2.32.3 # via # -c requirements/_base.txt # async-asgi-testclient # docker + # opentelemetry-exporter-otlp-proto-http respx==0.22.0 # via -r requirements/_test.in +rich==13.9.4 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/_base.in + # typer +rpds-py==0.23.1 + # via + # -c requirements/_base.txt + # jsonschema + # referencing +s3fs==2024.10.0 + # via fsspec s3transfer==0.11.3 # via boto3 +shellingham==1.5.4 + # via + # -c requirements/_base.txt + # typer six==1.17.0 # via # -c requirements/_base.txt @@ -288,18 +807,45 @@ sortedcontainers==2.4.0 sqlalchemy==1.4.54 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -r requirements/_test.in # alembic sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +stream-zip==0.0.83 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in tblib==3.0.0 # via # -c requirements/_base.txt # distributed +tenacity==9.0.0 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in toolz==1.0.0 # via # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in # dask # distributed # partd @@ -308,19 +854,40 @@ tornado==6.4.2 # -c requirements/_base.txt # bokeh # distributed +tqdm==4.67.1 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in +typer==0.15.2 + # via + # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/_base.in types-networkx==3.4.2.20250304 # via -r requirements/_test.in types-psycopg2==2.9.21.20250121 # via -r requirements/_test.in +types-python-dateutil==2.9.0.20241206 + # via + # -c requirements/_base.txt + # arrow types-pyyaml==6.0.12.20241230 # via -r requirements/_test.in typing-extensions==4.12.2 # via # -c requirements/_base.txt + # aiodebug # alembic # anyio + # faststream # mypy + # opentelemetry-sdk + # pydantic + # pydantic-core + # pydantic-extra-types # sqlalchemy2-stubs + # typer tzdata==2025.1 # via # faker @@ -328,6 +895,24 @@ tzdata==2025.1 urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # botocore # distributed @@ -337,11 +922,15 @@ wrapt==1.17.2 # via # -c requirements/_base.txt # aiobotocore + # deprecated + # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis xyzservices==2025.1.0 # via bokeh yarl==1.18.3 # via # -c requirements/_base.txt + # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq From 08438b9faae54ab643be985f72f52021351a0746 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 09:13:43 +0200 Subject: [PATCH 20/46] removed old parts --- .../modules/comp_scheduler/_scheduler_dask.py | 34 ------------------- .../modules/dask_client.py | 21 ------------ 2 files changed, 55 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py index 646512e21be0..72744aab6893 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py @@ -31,7 +31,6 @@ ) from ...models.comp_runs import CompRunsAtDB, Iteration, RunMetadataDict from ...models.comp_tasks import CompTaskAtDB -from ...models.dask_subsystem import DaskClientTaskState from ...utils.dask import ( clean_task_output_and_log_files_if_invalid, parse_output_data, @@ -50,18 +49,6 @@ _logger = logging.getLogger(__name__) -_DASK_CLIENT_TASK_STATE_TO_RUNNING_STATE_MAP: dict[ - DaskClientTaskState, RunningState -] = { - DaskClientTaskState.PENDING: RunningState.PENDING, - DaskClientTaskState.NO_WORKER: RunningState.WAITING_FOR_RESOURCES, - DaskClientTaskState.LOST: RunningState.UNKNOWN, - DaskClientTaskState.ERRED: RunningState.FAILED, - DaskClientTaskState.ABORTED: RunningState.ABORTED, - DaskClientTaskState.SUCCESS: RunningState.SUCCESS, -} - - @asynccontextmanager async def _cluster_dask_client( user_id: UserID, @@ -158,27 +145,6 @@ async def _get_tasks_status( run_metadata=comp_run.metadata, ) as client: return await client.get_tasks_status([f"{t.job_id}" for t in tasks]) - # tasks_statuses = await client.get_tasks_status( - # [f"{t.job_id}" for t in tasks] - # ) - # # process dask states - # running_states: list[RunningState] = [] - # for dask_task_state, task in zip(tasks_statuses, tasks, strict=True): - # if dask_task_state is DaskClientTaskState.PENDING_OR_STARTED: - # running_states += [ - # ( - # RunningState.STARTED - # if task.progress is not None - # else RunningState.PENDING - # ) - # ] - # else: - # running_states += [ - # _DASK_CLIENT_TASK_STATE_TO_RUNNING_STATE_MAP.get( - # dask_task_state, RunningState.UNKNOWN - # ) - # ] - # return running_states except ComputationalBackendOnDemandNotReadyError: _logger.info("The on demand computational backend is not ready yet...") diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index c6c5a87dc121..d02f1ddf926f 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -47,7 +47,6 @@ from dask_task_models_library.resource_constraints import ( create_ec2_resource_constraint_key, ) -from distributed.scheduler import TaskStateState as DaskSchedulerTaskState from fastapi import FastAPI from models_library.api_schemas_directorv2.clusters import ClusterDetails, Scheduler from models_library.clusters import ClusterAuthentication, ClusterTypeInModel @@ -77,7 +76,6 @@ from ..core.settings import AppSettings, ComputationalBackendSettings from ..models.comp_runs import RunMetadataDict from ..models.comp_tasks import Image -from ..models.dask_subsystem import DaskClientTaskState from ..modules.storage import StorageClient from ..utils import dask as dask_utils from ..utils.dask_client_utils import ( @@ -91,25 +89,6 @@ _logger = logging.getLogger(__name__) -# NOTE: processing does not mean the task is currently being computed, it means -# the task was accepted by a worker, but might still be queud in it -# see https://distributed.dask.org/en/stable/scheduling-state.html#task-state - - -_DASK_TASK_STATUS_DASK_CLIENT_TASK_STATE_MAP: dict[ - DaskSchedulerTaskState, DaskClientTaskState -] = { - "queued": DaskClientTaskState.PENDING, - "released": DaskClientTaskState.PENDING, - "waiting": DaskClientTaskState.PENDING, - "no-worker": DaskClientTaskState.NO_WORKER, - "processing": DaskClientTaskState.PENDING_OR_STARTED, - "memory": DaskClientTaskState.SUCCESS, - "erred": DaskClientTaskState.ERRED, - "forgotten": DaskClientTaskState.LOST, -} - - _DASK_DEFAULT_TIMEOUT_S: Final[int] = 5 From a62fd7f1245338880c1b4564283b6edac66d8a4e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 10:38:57 +0200 Subject: [PATCH 21/46] ensure plugins are loaded for tests --- .../src/pytest_simcore/dask_scheduler.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py b/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py index 03c63c07d348..938930958c58 100644 --- a/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py +++ b/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py @@ -19,6 +19,9 @@ def dask_workers_config() -> dict[str, Any]: "options": { "nthreads": 2, "resources": {"CPU": 2, "RAM": 48e9}, + "preload": ( + "simcore_service_dask_sidecar.task_life_cycle_worker_plugin", + ), }, }, "gpu-worker": { @@ -30,6 +33,9 @@ def dask_workers_config() -> dict[str, Any]: "GPU": 1, "RAM": 48e9, }, + "preload": ( + "simcore_service_dask_sidecar.task_life_cycle_worker_plugin", + ), }, }, "large-ram-worker": { @@ -40,6 +46,9 @@ def dask_workers_config() -> dict[str, Any]: "CPU": 8, "RAM": 768e9, }, + "preload": ( + "simcore_service_dask_sidecar.task_life_cycle_worker_plugin", + ), }, }, } @@ -54,6 +63,9 @@ def dask_scheduler_config( "options": { "port": unused_tcp_port_factory(), "dashboard_address": f":{unused_tcp_port_factory()}", + "preload": ( + "simcore_service_dask_sidecar.task_life_cycle_scheduler_plugin", + ), }, } From 0f8b59d51baf3a13a83ce06d93e3276af4923394 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 10:39:04 +0200 Subject: [PATCH 22/46] added dependency --- services/director-v2/requirements/dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/director-v2/requirements/dev.txt b/services/director-v2/requirements/dev.txt index f183201fd55a..94d010372d4d 100644 --- a/services/director-v2/requirements/dev.txt +++ b/services/director-v2/requirements/dev.txt @@ -21,5 +21,7 @@ --editable ../../packages/settings-library/ --editable ../../packages/simcore-sdk/ +--editable ../../services/dask-sidecar/ + # installs current package --editable . From 63ee2c1da986dd8748cd9c17f5ee8b7a7b56b6d3 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 10:39:18 +0200 Subject: [PATCH 23/46] allows to setup plugins --- .../task_life_cycle_scheduler_plugin.py | 7 +++++++ .../task_life_cycle_worker_plugin.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py index c0ac1ce7b1bf..8b2b5977155c 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py @@ -1,6 +1,7 @@ import logging from typing import Any +import click from dask.typing import Key from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState from distributed import Scheduler, SchedulerPlugin @@ -49,3 +50,9 @@ def transition( key, kwargs.get("worker"), finish ).model_dump(mode="json"), ) + + +@click.command() +def dask_setup(scheduler): + plugin = TaskLifecycleSchedulerPlugin() + scheduler.add_plugin(plugin) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py index ba07fa5836f3..84cb3d963f3e 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py @@ -2,6 +2,7 @@ from collections.abc import Awaitable from typing import Any +import click from dask.typing import Key from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState from distributed import WorkerPlugin @@ -51,3 +52,9 @@ def transition( key, kwargs.get("worker"), finish ).model_dump(mode="json"), ) + + +@click.command() +async def dask_setup(worker: Worker) -> None: + plugin = TaskLifecycleWorkerPlugin() + await worker.plugin_add(plugin) From 3f40d642a5ab4e25ef6ef48a1965f9ca7965be96 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 10:39:41 +0200 Subject: [PATCH 24/46] make the call async --- .../src/simcore_service_director_v2/modules/dask_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index d02f1ddf926f..7ef7beeb3045 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -506,7 +506,7 @@ async def release_task_result(self, job_id: str) -> None: # the variable was effectively deleted. # This is annoying as one can re-create the variable without error. var = distributed.Variable(job_id, client=self.backend.client) - var.delete() + await asyncio.get_event_loop().run_in_executor(None, var.delete) # first check if the key exists await dask_utils.wrap_client_async_routine( self.backend.client.get_dataset(name=job_id) From 4d6ea9c41b235058fc330f7e3cf8f96597ae3e50 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 10:40:02 +0200 Subject: [PATCH 25/46] the test now runs --- .../tests/unit/test_modules_dask_client.py | 63 +++++++++++-------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index 534dc034661b..ea9a55509129 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -6,6 +6,7 @@ # pylint: disable=reimported import asyncio import functools +import logging import traceback from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine from dataclasses import dataclass @@ -49,6 +50,7 @@ from models_library.users import UserID from pydantic import AnyUrl, ByteSize, TypeAdapter from pytest_mock.plugin import MockerFixture +from pytest_simcore.helpers.logging_tools import log_context from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.s3 import S3Settings from simcore_sdk.node_ports_v2 import FileLinkType @@ -140,41 +142,48 @@ def _minimal_dask_config( @pytest.fixture async def create_dask_client_from_scheduler( _minimal_dask_config: None, - dask_spec_local_cluster: SpecCluster, + dask_spec_local_cluster: distributed.SpecCluster, minimal_app: FastAPI, tasks_file_link_type: FileLinkType, ) -> AsyncIterator[Callable[[], Awaitable[DaskClient]]]: created_clients = [] async def factory() -> DaskClient: - client = await DaskClient.create( - app=minimal_app, - settings=minimal_app.state.settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND, - endpoint=TypeAdapter(AnyUrl).validate_python( - dask_spec_local_cluster.scheduler_address - ), - authentication=NoAuthentication(), - tasks_file_link_type=tasks_file_link_type, - cluster_type=ClusterTypeInModel.ON_PREMISE, - ) - assert client - assert client.app == minimal_app - assert ( - client.settings - == minimal_app.state.settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND - ) + with log_context( + logging.INFO, + f"Create director-v2 DaskClient to {dask_spec_local_cluster.scheduler_address}", + ) as ctx: + client = await DaskClient.create( + app=minimal_app, + settings=minimal_app.state.settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND, + endpoint=TypeAdapter(AnyUrl).validate_python( + dask_spec_local_cluster.scheduler_address + ), + authentication=NoAuthentication(), + tasks_file_link_type=tasks_file_link_type, + cluster_type=ClusterTypeInModel.ON_PREMISE, + ) + assert client + assert client.app == minimal_app + assert ( + client.settings + == minimal_app.state.settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND + ) + + assert client.backend.client + scheduler_infos = client.backend.client.scheduler_info() # type: ignore + ctx.logger.info( + "%s", + f"--> Connected to scheduler via client {client=} to scheduler {scheduler_infos=}", + ) - assert client.backend.client - scheduler_infos = client.backend.client.scheduler_info() # type: ignore - print( - f"--> Connected to scheduler via client {client=} to scheduler {scheduler_infos=}" - ) created_clients.append(client) return client yield factory - await asyncio.gather(*[client.delete() for client in created_clients]) - print(f"<-- Disconnected scheduler clients {created_clients=}") + + with log_context(logging.INFO, "Disconnect scheduler clients"): + await asyncio.gather(*[client.delete() for client in created_clients]) @pytest.fixture(params=["create_dask_client_from_scheduler"]) @@ -728,15 +737,17 @@ def fake_remote_fct( with pytest.raises(TaskCancelledError): await dask_client.get_task_result(published_computation_task[0].job_id) - # after releasing the results, the task shall be UNKNOWN + await asyncio.sleep(5) await dask_client.release_task_result(published_computation_task[0].job_id) + # after releasing the results, the task shall be UNKNOWN + # NOTE: this change of status takes a very long time to happen and is not relied upon so we skip it since it # makes the test fail a lot for no gain (it's kept here in case it ever becomes an issue) await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, RunningState.UNKNOWN, - timeout=120, + timeout=10, ) From 7e0faa6c7d02a3feb93741b99f98f8c6f45686e9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 10:53:59 +0200 Subject: [PATCH 26/46] set a timeout --- services/director-v2/tests/unit/test_modules_dask_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index ea9a55509129..909c3c238de7 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -737,10 +737,10 @@ def fake_remote_fct( with pytest.raises(TaskCancelledError): await dask_client.get_task_result(published_computation_task[0].job_id) - await asyncio.sleep(5) await dask_client.release_task_result(published_computation_task[0].job_id) # after releasing the results, the task shall be UNKNOWN - + _ALLOW_TIME_FOR_LOCAL_DASK_SCHEDULER_TO_UPDATE_TIMEOUT_S = 5 + await asyncio.sleep(_ALLOW_TIME_FOR_LOCAL_DASK_SCHEDULER_TO_UPDATE_TIMEOUT_S) # NOTE: this change of status takes a very long time to happen and is not relied upon so we skip it since it # makes the test fail a lot for no gain (it's kept here in case it ever becomes an issue) await _assert_wait_for_task_status( From 5cb2fc0fda312e79e926a82056e73f459ed86422 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 10:56:20 +0200 Subject: [PATCH 27/46] mypy --- .../src/simcore_service_director_v2/modules/dask_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index 7ef7beeb3045..dc980f5887e2 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -197,7 +197,7 @@ def _comp_sidecar_fct( ) -> TaskOutputData: """This function is serialized by the Dask client and sent over to the Dask sidecar(s) Therefore, (screaming here) DO NOT MOVE THAT IMPORT ANYWHERE ELSE EVER!!""" - from simcore_service_dask_sidecar.worker import ( # type: ignore[import-not-found] # this runs inside the dask-sidecar + from simcore_service_dask_sidecar.worker import ( # type: ignore[import-untyped] # this runs inside the dask-sidecar run_computational_sidecar, ) From 65bb64e5d5926a0da512eb476c6e400b807fd715 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 11:05:51 +0200 Subject: [PATCH 28/46] moved task life cycle plugins to dask-task-library --- .../src/dask_task_models_library/plugins/__init__.py | 0 .../plugins}/task_life_cycle_scheduler_plugin.py | 3 ++- .../plugins}/task_life_cycle_worker_plugin.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 packages/dask-task-models-library/src/dask_task_models_library/plugins/__init__.py rename {services/dask-sidecar/src/simcore_service_dask_sidecar => packages/dask-task-models-library/src/dask_task_models_library/plugins}/task_life_cycle_scheduler_plugin.py (94%) rename {services/dask-sidecar/src/simcore_service_dask_sidecar => packages/dask-task-models-library/src/dask_task_models_library/plugins}/task_life_cycle_worker_plugin.py (95%) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/plugins/__init__.py b/packages/dask-task-models-library/src/dask_task_models_library/plugins/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py similarity index 94% rename from services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py rename to packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py index 8b2b5977155c..1bee5d677e7c 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_scheduler_plugin.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py @@ -3,11 +3,12 @@ import click from dask.typing import Key -from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState from distributed import Scheduler, SchedulerPlugin from distributed.scheduler import TaskStateState from servicelib.logging_utils import log_context +from ..models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState + _logger = logging.getLogger(__name__) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_worker_plugin.py similarity index 95% rename from services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py rename to packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_worker_plugin.py index 84cb3d963f3e..235b21136985 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/task_life_cycle_worker_plugin.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_worker_plugin.py @@ -4,12 +4,13 @@ import click from dask.typing import Key -from dask_task_models_library.models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState from distributed import WorkerPlugin from distributed.worker import Worker from distributed.worker_state_machine import TaskStateState from servicelib.logging_utils import log_context +from ..models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState + _logger = logging.getLogger(__name__) From 65262ca48b27fc6ad74a87f34be2d80fafd2f30e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 11:06:38 +0200 Subject: [PATCH 29/46] removed dependency to dask-sidecar --- services/director-v2/requirements/_test.in | 2 - services/director-v2/requirements/_test.txt | 593 +------------------- services/director-v2/requirements/dev.txt | 2 - 3 files changed, 2 insertions(+), 595 deletions(-) diff --git a/services/director-v2/requirements/_test.in b/services/director-v2/requirements/_test.in index 8e2bf22b23bb..2fb831189bae 100644 --- a/services/director-v2/requirements/_test.in +++ b/services/director-v2/requirements/_test.in @@ -9,8 +9,6 @@ # --constraint _base.txt ---requirement ../../../services/dask-sidecar/requirements/_base.in # getting access to plugins - aio_pika aioboto3 alembic # migration due to pytest_simcore.postgres_service2 diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index 2bc9d5bb7d99..f28a0daa1544 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -1,32 +1,14 @@ aio-pika==9.5.5 # via # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_test.in aioboto3==14.3.0 # via -r requirements/_test.in aiobotocore==2.22.0 - # via - # aioboto3 - # s3fs -aiocache==0.12.3 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in -aiodebug==2.3.0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.24.0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/_base.in + # via aioboto3 aiofiles==24.1.0 # via # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/_base.in # aioboto3 aiohappyeyeballs==2.5.0 # via @@ -35,29 +17,8 @@ aiohappyeyeballs==2.5.0 aiohttp==3.11.18 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aiobotocore - # aiodocker - # fsspec - # s3fs aioitertools==0.12.0 # via aiobotocore aiormq==6.8.1 @@ -72,23 +33,10 @@ alembic==1.15.1 # via # -c requirements/_base.txt # -r requirements/_test.in -annotated-types==0.7.0 - # via - # -c requirements/_base.txt - # pydantic anyio==4.8.0 # via # -c requirements/_base.txt - # fast-depends - # faststream # httpx -arrow==1.3.0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in asgi-lifespan==2.1.0 # via -r requirements/_test.in async-asgi-testclient==1.4.11 @@ -97,13 +45,7 @@ attrs==25.1.0 # via # -c requirements/_base.txt # aiohttp - # jsonschema # pytest-docker - # referencing -blosc==1.11.2 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/_base.in bokeh==3.6.3 # via dask boto3==1.37.3 @@ -116,24 +58,6 @@ botocore==1.37.3 certifi==2025.1.31 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # httpcore # httpx @@ -147,7 +71,6 @@ click==8.1.7 # -c requirements/_base.txt # dask # distributed - # typer cloudpickle==3.1.0 # via # -c requirements/_base.txt @@ -159,33 +82,15 @@ coverage==7.6.12 # via pytest-cov dask==2024.12.0 # via - # -c requirements/../../../services/dask-sidecar/requirements/constraints.txt # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/_base.in # -r requirements/_test.in # distributed -deprecated==1.2.18 - # via - # -c requirements/_base.txt - # opentelemetry-api - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-semantic-conventions distributed==2024.12.0 # via # -c requirements/_base.txt # dask -dnspython==2.7.0 - # via - # -c requirements/_base.txt - # email-validator docker==7.1.0 # via -r requirements/_test.in -email-validator==2.2.0 - # via - # -c requirements/_base.txt - # pydantic exceptiongroup==1.2.2 # via # -c requirements/_base.txt @@ -194,14 +99,6 @@ execnet==2.1.1 # via pytest-xdist faker==36.2.2 # via -r requirements/_test.in -fast-depends==2.4.12 - # via - # -c requirements/_base.txt - # faststream -faststream==0.5.35 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in flaky==3.8.1 # via -r requirements/_test.in frozenlist==1.5.0 @@ -212,22 +109,11 @@ frozenlist==1.5.0 fsspec==2024.10.0 # via # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/_base.in # dask - # s3fs -googleapis-common-protos==1.69.1 - # via - # -c requirements/_base.txt - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http greenlet==3.1.1 # via # -c requirements/_base.txt # sqlalchemy -grpcio==1.70.0 - # via - # -c requirements/_base.txt - # opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # -c requirements/_base.txt @@ -239,24 +125,6 @@ httpcore==1.0.7 httpx==0.28.1 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # respx icdiff==2.0.7 @@ -265,7 +133,6 @@ idna==3.10 # via # -c requirements/_base.txt # anyio - # email-validator # httpx # requests # yarl @@ -273,30 +140,11 @@ importlib-metadata==8.5.0 # via # -c requirements/_base.txt # dask - # opentelemetry-api iniconfig==2.0.0 # via pytest jinja2==3.1.4 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # bokeh # dask @@ -306,61 +154,21 @@ jmespath==1.0.1 # aiobotocore # boto3 # botocore -jsonschema==4.23.0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -jsonschema-specifications==2024.10.1 - # via - # -c requirements/_base.txt - # jsonschema locket==1.0.0 # via # -c requirements/_base.txt # distributed # partd -lz4==4.3.3 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/_base.in mako==1.3.9 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # alembic -markdown-it-py==3.0.0 - # via - # -c requirements/_base.txt - # rich markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 # mako -mdurl==0.1.2 - # via - # -c requirements/_base.txt - # markdown-it-py msgpack==1.1.0 # via # -c requirements/_base.txt @@ -383,117 +191,12 @@ numpy==2.1.3 # contourpy # pandas # types-networkx -opentelemetry-api==1.30.0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-instrumentation - # opentelemetry-instrumentation-logging - # opentelemetry-instrumentation-redis - # opentelemetry-instrumentation-requests - # opentelemetry-sdk - # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.30.0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.30.0 - # via - # -c requirements/_base.txt - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.30.0 - # via - # -c requirements/_base.txt - # opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.30.0 - # via - # -c requirements/_base.txt - # opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.51b0 - # via - # -c requirements/_base.txt - # opentelemetry-instrumentation-logging - # opentelemetry-instrumentation-redis - # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-logging==0.51b0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.51b0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.51b0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.30.0 - # via - # -c requirements/_base.txt - # opentelemetry-exporter-otlp-proto-common - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.30.0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.51b0 - # via - # -c requirements/_base.txt - # opentelemetry-instrumentation - # opentelemetry-instrumentation-redis - # opentelemetry-instrumentation-requests - # opentelemetry-sdk -opentelemetry-util-http==0.51b0 - # via - # -c requirements/_base.txt - # opentelemetry-instrumentation-requests -orjson==3.10.15 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.2 # via # -c requirements/_base.txt # bokeh # dask # distributed - # opentelemetry-instrumentation # pytest pamqp==3.3.0 # via @@ -511,127 +214,15 @@ pluggy==1.5.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff -prometheus-client==0.21.1 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/_base.in propcache==0.3.0 # via # -c requirements/_base.txt # aiohttp # yarl -protobuf==5.29.3 - # via - # -c requirements/_base.txt - # googleapis-common-protos - # opentelemetry-proto psutil==6.1.0 # via # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in # distributed -pycryptodome==3.21.0 - # via - # -c requirements/_base.txt - # stream-zip -pydantic==2.10.6 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/_base.in - # fast-depends - # pydantic-extra-types - # pydantic-settings -pydantic-core==2.27.2 - # via - # -c requirements/_base.txt - # pydantic -pydantic-extra-types==2.10.2 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -pydantic-settings==2.7.0 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/_base.in -pygments==2.19.1 - # via - # -c requirements/_base.txt - # rich -pyinstrument==5.0.1 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in pytest==8.3.5 # via # -r requirements/_test.in @@ -644,24 +235,6 @@ pytest==8.3.5 pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -r requirements/_test.in pytest-cov==6.0.0 # via -r requirements/_test.in @@ -679,118 +252,26 @@ python-dateutil==2.9.0.post0 # via # -c requirements/_base.txt # aiobotocore - # arrow # botocore # pandas -python-dotenv==1.0.1 - # via - # -c requirements/_base.txt - # pydantic-settings pytz==2025.1 # via pandas pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in + # -c requirements/_base.txt # bokeh # dask # distributed -redis==5.2.1 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in -referencing==0.35.1 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # jsonschema - # jsonschema-specifications -repro-zipfile==0.4.0 - # via -r requirements/../../../services/dask-sidecar/requirements/_base.in requests==2.32.3 # via # -c requirements/_base.txt # async-asgi-testclient # docker - # opentelemetry-exporter-otlp-proto-http respx==0.22.0 # via -r requirements/_test.in -rich==13.9.4 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/_base.in - # typer -rpds-py==0.23.1 - # via - # -c requirements/_base.txt - # jsonschema - # referencing -s3fs==2024.10.0 - # via fsspec s3transfer==0.11.3 # via boto3 -shellingham==1.5.4 - # via - # -c requirements/_base.txt - # typer six==1.17.0 # via # -c requirements/_base.txt @@ -807,45 +288,18 @@ sortedcontainers==2.4.0 sqlalchemy==1.4.54 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -r requirements/_test.in # alembic sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy -stream-zip==0.0.83 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in tblib==3.0.0 # via # -c requirements/_base.txt # distributed -tenacity==9.0.0 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in toolz==1.0.0 # via # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in # dask # distributed # partd @@ -854,40 +308,19 @@ tornado==6.4.2 # -c requirements/_base.txt # bokeh # distributed -tqdm==4.67.1 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in -typer==0.15.2 - # via - # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/_base.in types-networkx==3.4.2.20250304 # via -r requirements/_test.in types-psycopg2==2.9.21.20250121 # via -r requirements/_test.in -types-python-dateutil==2.9.0.20241206 - # via - # -c requirements/_base.txt - # arrow types-pyyaml==6.0.12.20241230 # via -r requirements/_test.in typing-extensions==4.12.2 # via # -c requirements/_base.txt - # aiodebug # alembic # anyio - # faststream # mypy - # opentelemetry-sdk - # pydantic - # pydantic-core - # pydantic-extra-types # sqlalchemy2-stubs - # typer tzdata==2025.1 # via # faker @@ -895,24 +328,6 @@ tzdata==2025.1 urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../services/dask-sidecar/requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # botocore # distributed @@ -922,15 +337,11 @@ wrapt==1.17.2 # via # -c requirements/_base.txt # aiobotocore - # deprecated - # opentelemetry-instrumentation - # opentelemetry-instrumentation-redis xyzservices==2025.1.0 # via bokeh yarl==1.18.3 # via # -c requirements/_base.txt - # -r requirements/../../../services/dask-sidecar/requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/director-v2/requirements/dev.txt b/services/director-v2/requirements/dev.txt index 94d010372d4d..f183201fd55a 100644 --- a/services/director-v2/requirements/dev.txt +++ b/services/director-v2/requirements/dev.txt @@ -21,7 +21,5 @@ --editable ../../packages/settings-library/ --editable ../../packages/simcore-sdk/ ---editable ../../services/dask-sidecar/ - # installs current package --editable . From d5e4314505d72e2b38c280541bf29105bac7341c Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 11:17:58 +0200 Subject: [PATCH 30/46] fixed paths --- .../src/simcore_service_dask_sidecar/scheduler.py | 6 +++--- .../dask-sidecar/src/simcore_service_dask_sidecar/worker.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py index 31ca65b944f0..0813da017416 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py @@ -1,13 +1,13 @@ import logging import distributed +from dask_task_models_library.plugins.task_life_cycle_scheduler_plugin import ( + TaskLifecycleSchedulerPlugin, +) from servicelib.logging_utils import log_context from ._meta import print_dask_scheduler_banner from .settings import ApplicationSettings -from .task_life_cycle_scheduler_plugin import ( - TaskLifecycleSchedulerPlugin, -) from .utils.logs import setup_app_logging _logger = logging.getLogger(__name__) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py index 41b049e2c908..abaedd698ed6 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py @@ -11,6 +11,9 @@ ContainerTaskParameters, LogFileUploadURL, ) +from dask_task_models_library.plugins.task_life_cycle_worker_plugin import ( + TaskLifecycleWorkerPlugin, +) from servicelib.logging_utils import log_context from settings_library.s3 import S3Settings @@ -18,9 +21,6 @@ from .computational_sidecar.core import ComputationalSidecar from .rabbitmq_worker_plugin import RabbitMQPlugin from .settings import ApplicationSettings -from .task_life_cycle_worker_plugin import ( - TaskLifecycleWorkerPlugin, -) from .utils.dask import ( TaskPublisher, get_current_task_resources, From c9b3698a3d575da45594b3bb6e8fd38ad2775711 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 11:21:35 +0200 Subject: [PATCH 31/46] moved plugins down one level --- .../pytest-simcore/src/pytest_simcore/dask_scheduler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py b/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py index 938930958c58..54fdd0212231 100644 --- a/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py +++ b/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py @@ -20,7 +20,7 @@ def dask_workers_config() -> dict[str, Any]: "nthreads": 2, "resources": {"CPU": 2, "RAM": 48e9}, "preload": ( - "simcore_service_dask_sidecar.task_life_cycle_worker_plugin", + "dask_task_models_library.plugins.task_life_cycle_worker_plugin", ), }, }, @@ -34,7 +34,7 @@ def dask_workers_config() -> dict[str, Any]: "RAM": 48e9, }, "preload": ( - "simcore_service_dask_sidecar.task_life_cycle_worker_plugin", + "dask_task_models_library.plugins.task_life_cycle_worker_plugin", ), }, }, @@ -47,7 +47,7 @@ def dask_workers_config() -> dict[str, Any]: "RAM": 768e9, }, "preload": ( - "simcore_service_dask_sidecar.task_life_cycle_worker_plugin", + "dask_task_models_library.plugins.task_life_cycle_worker_plugin", ), }, }, @@ -64,7 +64,7 @@ def dask_scheduler_config( "port": unused_tcp_port_factory(), "dashboard_address": f":{unused_tcp_port_factory()}", "preload": ( - "simcore_service_dask_sidecar.task_life_cycle_scheduler_plugin", + "dask_task_models_library.plugins.task_life_cycle_scheduler_plugin", ), }, } From 4084721c1aac0ee1eda9a4638a3e2b4b2accdef6 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 14:54:38 +0200 Subject: [PATCH 32/46] manage to poll task progress now --- .../modules/comp_scheduler/_scheduler_base.py | 62 ++++++++++++++----- .../modules/comp_scheduler/_scheduler_dask.py | 57 +++++++++++++++++ .../modules/dask_client.py | 29 ++++++++- 3 files changed, 129 insertions(+), 19 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py index 49465084d5d7..c24a8eab5ec4 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py @@ -116,11 +116,11 @@ class SortedTasks: async def _triage_changed_tasks( - changed_tasks: list[tuple[_Previous, _Current]], + changed_tasks_or_executing: list[tuple[_Previous, _Current]], ) -> SortedTasks: started_tasks = [ current - for previous, current in changed_tasks + for previous, current in changed_tasks_or_executing if current.state in RUNNING_STATES or ( previous.state in WAITING_FOR_START_STATES @@ -130,17 +130,21 @@ async def _triage_changed_tasks( # NOTE: some tasks can be both started and completed since we might have the time they were running completed_tasks = [ - current for _, current in changed_tasks if current.state in COMPLETED_STATES + current + for _, current in changed_tasks_or_executing + if current.state in COMPLETED_STATES ] waiting_for_resources_tasks = [ current - for previous, current in changed_tasks + for previous, current in changed_tasks_or_executing if current.state in WAITING_FOR_START_STATES ] lost_or_momentarily_lost_tasks = [ - current for _, current in changed_tasks if current.state is RunningState.UNKNOWN + current + for _, current in changed_tasks_or_executing + if current.state is RunningState.UNKNOWN ] if lost_or_momentarily_lost_tasks: _logger.warning( @@ -321,21 +325,30 @@ async def _get_changed_tasks_from_backend( user_id: UserID, processing_tasks: list[CompTaskAtDB], comp_run: CompRunsAtDB, - ) -> list[tuple[_Previous, _Current]]: + ) -> tuple[list[tuple[_Previous, _Current]], list[CompTaskAtDB]]: tasks_backend_status = await self._get_tasks_status( user_id, processing_tasks, comp_run ) - return [ - ( - task, - task.model_copy(update={"state": backend_state}), - ) - for task, backend_state in zip( - processing_tasks, tasks_backend_status, strict=True - ) - if task.state is not backend_state - ] + return ( + [ + ( + task, + task.model_copy(update={"state": backend_state}), + ) + for task, backend_state in zip( + processing_tasks, tasks_backend_status, strict=True + ) + if task.state is not backend_state + ], + [ + task + for task, backend_state in zip( + processing_tasks, tasks_backend_status, strict=True + ) + if task.state is backend_state is RunningState.STARTED + ], + ) async def _process_started_tasks( self, @@ -476,7 +489,10 @@ async def _update_states_from_comp_backend( return # get the tasks which state actually changed since last check - tasks_with_changed_states = await self._get_changed_tasks_from_backend( + ( + tasks_with_changed_states, + executing_tasks, + ) = await self._get_changed_tasks_from_backend( user_id, tasks_inprocess, comp_run ) # NOTE: typical states a task goes through @@ -511,6 +527,9 @@ async def _update_states_from_comp_backend( if sorted_tasks.waiting: await self._process_waiting_tasks(sorted_tasks.waiting) + if executing_tasks: + await self._process_executing_tasks(user_id, executing_tasks, comp_run) + @abstractmethod async def _start_tasks( self, @@ -545,6 +564,15 @@ async def _process_completed_tasks( ) -> None: """process tasks from the 3rd party backend""" + @abstractmethod + async def _process_executing_tasks( + self, + user_id: UserID, + tasks: list[CompTaskAtDB], + comp_run: CompRunsAtDB, + ) -> None: + """process executing tasks from the 3rd party backend""" + async def apply( self, *, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py index 72744aab6893..bb4a85e446de 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py @@ -37,6 +37,7 @@ ) from ...utils.dask_client_utils import TaskHandlers, UnixTimestamp from ...utils.rabbitmq import ( + publish_service_progress, publish_service_resource_tracking_stopped, publish_service_stopped_metrics, ) @@ -150,6 +151,62 @@ async def _get_tasks_status( _logger.info("The on demand computational backend is not ready yet...") return [RunningState.WAITING_FOR_CLUSTER] * len(tasks) + async def _process_executing_tasks( + self, + user_id: UserID, + tasks: list[CompTaskAtDB], + comp_run: CompRunsAtDB, + ) -> None: + task_progresses = [] + try: + async with _cluster_dask_client( + user_id, + self, + use_on_demand_clusters=comp_run.use_on_demand_clusters, + run_metadata=comp_run.metadata, + ) as client: + task_progresses = await client.get_tasks_progress( + [f"{t.job_id}" for t in tasks], + ) + for task_progress_event in task_progresses: + if task_progress_event: + await CompTasksRepository( + self.db_engine + ).update_project_task_progress( + task_progress_event.task_owner.project_id, + task_progress_event.task_owner.node_id, + task_progress_event.progress, + ) + + except ComputationalBackendOnDemandNotReadyError: + _logger.info("The on demand computational backend is not ready yet...") + + comp_tasks_repo = CompTasksRepository(self.db_engine) + await asyncio.gather( + *( + comp_tasks_repo.update_project_task_progress( + t.task_owner.project_id, + t.task_owner.node_id, + t.progress, + ) + for t in task_progresses + if t + ) + ) + await asyncio.gather( + *( + publish_service_progress( + self.rabbitmq_client, + user_id=t.task_owner.user_id, + project_id=t.task_owner.project_id, + node_id=t.task_owner.node_id, + progress=t.progress, + ) + for t in task_progresses + if t + ) + ) + async def _stop_tasks( self, user_id: UserID, tasks: list[CompTaskAtDB], comp_run: CompRunsAtDB ) -> None: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index dc980f5887e2..76eccfc630f5 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -405,6 +405,33 @@ async def send_computation_tasks( return list_of_node_id_to_job_id + async def get_tasks_progress( + self, job_ids: list[str] + ) -> tuple[TaskProgressEvent | None, ...]: + dask_utils.check_scheduler_is_still_the_same( + self.backend.scheduler_id, self.backend.client + ) + dask_utils.check_communication_with_scheduler_is_open(self.backend.client) + dask_utils.check_scheduler_status(self.backend.client) + + dask_events = await self.backend.client.get_events( + TaskProgressEvent.topic_name() + ) + + if not dask_events: + return tuple([None] * len(job_ids)) + last_task_progress = [] + for job_id in job_ids: + progress_event = None + for dask_event in reversed(dask_events): + parsed_event = TaskProgressEvent.model_validate_json(dask_event[1]) + if parsed_event.job_id == job_id: + progress_event = parsed_event + break + last_task_progress.append(progress_event) + + return tuple(last_task_progress) + async def get_tasks_status(self, job_ids: Iterable[str]) -> list[RunningState]: dask_utils.check_scheduler_is_still_the_same( self.backend.scheduler_id, self.backend.client @@ -413,8 +440,6 @@ async def get_tasks_status(self, job_ids: Iterable[str]) -> list[RunningState]: dask_utils.check_scheduler_status(self.backend.client) async def _get_job_id_status(job_id: str) -> RunningState: - # TODO: maybe we should define an event just for that, instead of multiple calls - # but the max length by default is 1000. We should test it dask_events: tuple[tuple[UnixTimestamp, str], ...] = ( await self.backend.client.get_events( TASK_LIFE_CYCLE_EVENT.format(key=job_id) From 3e3aab15047e7aa7001713bd70fd9b70020d3fca Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 14:57:42 +0200 Subject: [PATCH 33/46] revert to have events --- .../modules/comp_scheduler/_scheduler_dask.py | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py index bb4a85e446de..0c7cbd6523eb 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py @@ -44,6 +44,9 @@ from ..clusters_keeper import get_or_create_on_demand_cluster from ..dask_client import DaskClient, PublishedComputationTask from ..dask_clients_pool import DaskClientsPool +from ..db.repositories.comp_runs import ( + CompRunsRepository, +) from ..db.repositories.comp_tasks import CompTasksRepository from ._scheduler_base import BaseCompScheduler @@ -373,30 +376,30 @@ async def _task_progress_change_handler( with log_catch(_logger, reraise=False): task_progress_event = TaskProgressEvent.model_validate_json(event[1]) _logger.debug("received task progress update: %s", task_progress_event) - # user_id = task_progress_event.task_owner.user_id - # project_id = task_progress_event.task_owner.project_id - # node_id = task_progress_event.task_owner.node_id - # comp_tasks_repo = CompTasksRepository(self.db_engine) - # task = await comp_tasks_repo.get_task(project_id, node_id) - # if task.progress is None: - # task.state = RunningState.STARTED - # task.progress = task_progress_event.progress - # run = await CompRunsRepository(self.db_engine).get(user_id, project_id) - # await self._process_started_tasks( - # [task], - # user_id=user_id, - # project_id=project_id, - # iteration=run.iteration, - # run_metadata=run.metadata, - # ) - # else: - # await comp_tasks_repo.update_project_task_progress( - # project_id, node_id, task_progress_event.progress - # ) - # await publish_service_progress( - # self.rabbitmq_client, - # user_id=user_id, - # project_id=project_id, - # node_id=node_id, - # progress=task_progress_event.progress, - # ) + user_id = task_progress_event.task_owner.user_id + project_id = task_progress_event.task_owner.project_id + node_id = task_progress_event.task_owner.node_id + comp_tasks_repo = CompTasksRepository(self.db_engine) + task = await comp_tasks_repo.get_task(project_id, node_id) + if task.progress is None: + task.state = RunningState.STARTED + task.progress = task_progress_event.progress + run = await CompRunsRepository(self.db_engine).get(user_id, project_id) + await self._process_started_tasks( + [task], + user_id=user_id, + project_id=project_id, + iteration=run.iteration, + run_metadata=run.metadata, + ) + else: + await comp_tasks_repo.update_project_task_progress( + project_id, node_id, task_progress_event.progress + ) + await publish_service_progress( + self.rabbitmq_client, + user_id=user_id, + project_id=project_id, + node_id=node_id, + progress=task_progress_event.progress, + ) From eb4debbf3249e250ce5d5700538add8d7c976add Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 15:01:45 +0200 Subject: [PATCH 34/46] update notes --- .../modules/comp_scheduler/_scheduler_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py index c24a8eab5ec4..7473eec54fd0 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py @@ -504,9 +504,10 @@ async def _update_states_from_comp_backend( # now process the tasks if sorted_tasks.started: # NOTE: the dask-scheduler cannot differentiate between tasks that are effectively computing and - # tasks that are only queued and accepted by a dask-worker. + # tasks that are only queued and accepted by a dask-worker. We use dask plugins to report on tasks states + # states are published to log_event, and we directly publish into RabbitMQ the sidecar and services logs. # tasks_started should therefore be mostly empty but for cases where - # - dask Pub/Sub mechanism failed, the tasks goes from PENDING -> SUCCESS/FAILED/ABORTED without STARTED + # - dask log_event/subscribe_topic mechanism failed, the tasks goes from PENDING -> SUCCESS/FAILED/ABORTED without STARTED # - the task finished so fast that the STARTED state was skipped between 2 runs of the dv-2 comp scheduler await self._process_started_tasks( sorted_tasks.started, From 205c96899ab31c06ec57afc1cb3aa3b5d1f98de0 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 15:14:03 +0200 Subject: [PATCH 35/46] added specific events to be more effective --- .../src/dask_task_models_library/models.py | 1 + .../utils/dask.py | 9 ++++- .../modules/dask_client.py | 34 ++++++++----------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/models.py b/packages/dask-task-models-library/src/dask_task_models_library/models.py index 42f961bd3b8c..2f701440d1d5 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/models.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/models.py @@ -10,6 +10,7 @@ DaskResources: TypeAlias = dict[str, int | float] TASK_LIFE_CYCLE_EVENT: Final[str] = "task-lifecycle-{key}" +TASK_RUNNING_PROGRESS_EVENT: Final[str] = "task-progress-{key}" _SCHEDULER_TASK_STATE_TO_RUNNING_STATE: Final[ dict[SchedulerTaskState, RunningState] ] = { diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/dask.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/dask.py index 116f3a70adfe..60e84053ed0f 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/dask.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/dask.py @@ -13,6 +13,7 @@ ) from dask_task_models_library.container_tasks.io import TaskCancelEventName from dask_task_models_library.container_tasks.protocol import TaskOwner +from dask_task_models_library.models import TASK_RUNNING_PROGRESS_EVENT from distributed.worker import get_worker from distributed.worker_state_machine import TaskState from models_library.progress_bar import ProgressReport @@ -178,4 +179,10 @@ def publish_event( log_catch(_logger, reraise=False), log_context(_logger, logging.DEBUG, msg=f"publishing {event=}"), ): - worker.log_event(TaskProgressEvent.topic_name(), event.model_dump_json()) + worker.log_event( + [ + TaskProgressEvent.topic_name(), + TASK_RUNNING_PROGRESS_EVENT.format(key=event.job_id), + ], + event.model_dump_json(), + ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index 76eccfc630f5..a07e83aa52e2 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -40,6 +40,7 @@ from dask_task_models_library.container_tasks.utils import generate_dask_job_id from dask_task_models_library.models import ( TASK_LIFE_CYCLE_EVENT, + TASK_RUNNING_PROGRESS_EVENT, DaskJobID, DaskResources, TaskLifeCycleState, @@ -407,30 +408,25 @@ async def send_computation_tasks( async def get_tasks_progress( self, job_ids: list[str] - ) -> tuple[TaskProgressEvent | None, ...]: + ) -> list[TaskProgressEvent | None]: dask_utils.check_scheduler_is_still_the_same( self.backend.scheduler_id, self.backend.client ) dask_utils.check_communication_with_scheduler_is_open(self.backend.client) dask_utils.check_scheduler_status(self.backend.client) - dask_events = await self.backend.client.get_events( - TaskProgressEvent.topic_name() - ) - - if not dask_events: - return tuple([None] * len(job_ids)) - last_task_progress = [] - for job_id in job_ids: - progress_event = None - for dask_event in reversed(dask_events): - parsed_event = TaskProgressEvent.model_validate_json(dask_event[1]) - if parsed_event.job_id == job_id: - progress_event = parsed_event - break - last_task_progress.append(progress_event) + async def _get_task_progress(job_id: str) -> TaskProgressEvent | None: + dask_events: tuple[tuple[UnixTimestamp, str], ...] = ( + await self.backend.client.get_events( + TASK_RUNNING_PROGRESS_EVENT.format(key=job_id) + ) + ) + if not dask_events: + return None + # we are interested in the last event + return TaskProgressEvent.model_validate_json(dask_events[-1][1]) - return tuple(last_task_progress) + return await asyncio.gather(*(_get_task_progress(job_id) for job_id in job_ids)) async def get_tasks_status(self, job_ids: Iterable[str]) -> list[RunningState]: dask_utils.check_scheduler_is_still_the_same( @@ -439,7 +435,7 @@ async def get_tasks_status(self, job_ids: Iterable[str]) -> list[RunningState]: dask_utils.check_communication_with_scheduler_is_open(self.backend.client) dask_utils.check_scheduler_status(self.backend.client) - async def _get_job_id_status(job_id: str) -> RunningState: + async def _get_task_state(job_id: str) -> RunningState: dask_events: tuple[tuple[UnixTimestamp, str], ...] = ( await self.backend.client.get_events( TASK_LIFE_CYCLE_EVENT.format(key=job_id) @@ -480,7 +476,7 @@ async def _get_job_id_status(job_id: str) -> RunningState: return parsed_event.state - return await asyncio.gather(*(_get_job_id_status(job_id) for job_id in job_ids)) + return await asyncio.gather(*(_get_task_state(job_id) for job_id in job_ids)) async def abort_computation_task(self, job_id: str) -> None: # Dask future may be cancelled, but only a future that was not already taken by From 443f5969e0e92bea84bcd478df0af428c3b90321 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 16:19:06 +0200 Subject: [PATCH 36/46] remove servicelib from dask-task-models --- .../task_life_cycle_scheduler_plugin.py | 46 ++++++++----------- .../plugins/task_life_cycle_worker_plugin.py | 37 +++++---------- 2 files changed, 32 insertions(+), 51 deletions(-) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py index 1bee5d677e7c..4e0a338abc9b 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py @@ -5,7 +5,6 @@ from dask.typing import Key from distributed import Scheduler, SchedulerPlugin from distributed.scheduler import TaskStateState -from servicelib.logging_utils import log_context from ..models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState @@ -14,20 +13,12 @@ class TaskLifecycleSchedulerPlugin(SchedulerPlugin): def __init__(self) -> None: - with log_context( - _logger, - logging.INFO, - "TaskLifecycleSchedulerPlugin init", - ): - self.scheduler = None + self.scheduler = None + _logger.info("initialized TaskLifecycleSchedulerPlugin") async def start(self, scheduler: Scheduler) -> None: - with log_context( - _logger, - logging.INFO, - "TaskLifecycleSchedulerPlugin start", - ): - self.scheduler = scheduler # type: ignore[assignment] + self.scheduler = scheduler # type: ignore[assignment] + _logger.info("started TaskLifecycleSchedulerPlugin") def transition( self, @@ -38,19 +29,22 @@ def transition( stimulus_id: str, **kwargs: Any, ): - with log_context( - _logger, - logging.INFO, - f"Task {key!r} transition from {start} to {finish} due to {stimulus_id=}", - ): - assert self.scheduler # nosec - - self.scheduler.log_event( - TASK_LIFE_CYCLE_EVENT.format(key=key), - TaskLifeCycleState.from_scheduler_task_state( - key, kwargs.get("worker"), finish - ).model_dump(mode="json"), - ) + _logger.debug( + "Task %s transition from %s to %s due to %s", + key, + start, + finish, + stimulus_id, + ) + + assert self.scheduler # nosec + + self.scheduler.log_event( + TASK_LIFE_CYCLE_EVENT.format(key=key), + TaskLifeCycleState.from_scheduler_task_state( + key, kwargs.get("worker"), finish + ).model_dump(mode="json"), + ) @click.command() diff --git a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_worker_plugin.py b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_worker_plugin.py index 235b21136985..ebc6aabcad85 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_worker_plugin.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_worker_plugin.py @@ -7,7 +7,6 @@ from distributed import WorkerPlugin from distributed.worker import Worker from distributed.worker_state_machine import TaskStateState -from servicelib.logging_utils import log_context from ..models import TASK_LIFE_CYCLE_EVENT, TaskLifeCycleState @@ -16,21 +15,13 @@ class TaskLifecycleWorkerPlugin(WorkerPlugin): def __init__(self) -> None: - with log_context( - _logger, - logging.INFO, - "TaskLifecycleWorkerPlugin init", - ): - self._worker = None + self._worker = None + _logger.info("TaskLifecycleWorkerPlugin initialized") def setup(self, worker: Worker) -> Awaitable[None]: async def _() -> None: - with log_context( - _logger, - logging.INFO, - "TaskLifecycleWorkerPlugin start", - ): - self._worker = worker # type: ignore[assignment] + self._worker = worker # type: ignore[assignment] + _logger.info("TaskLifecycleWorkerPlugin setup completed") return _() @@ -41,18 +32,14 @@ def transition( finish: TaskStateState, **kwargs: Any, ): - with log_context( - _logger, - logging.INFO, - f"Task {key!r} transition from {start} to {finish}", - ): - assert self._worker # nosec - self._worker.log_event( - TASK_LIFE_CYCLE_EVENT.format(key=key), - TaskLifeCycleState.from_worker_task_state( - key, kwargs.get("worker"), finish - ).model_dump(mode="json"), - ) + _logger.info("Task '%s' transition from %s to %s", key, start, finish) + assert self._worker # nosec + self._worker.log_event( + TASK_LIFE_CYCLE_EVENT.format(key=key), + TaskLifeCycleState.from_worker_task_state( + key, kwargs.get("worker"), finish + ).model_dump(mode="json"), + ) @click.command() From 27d832cd3ccdb2cddac8c43c24063051eed29779 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 16:21:20 +0200 Subject: [PATCH 37/46] added dependency for testing --- services/clusters-keeper/requirements/ci.txt | 1 + services/clusters-keeper/requirements/dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/services/clusters-keeper/requirements/ci.txt b/services/clusters-keeper/requirements/ci.txt index 7bb3a4afb29f..22fd83a2698c 100644 --- a/services/clusters-keeper/requirements/ci.txt +++ b/services/clusters-keeper/requirements/ci.txt @@ -14,6 +14,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library simcore-common-library @ ../../packages/common-library +simcore-dask-task-models-library @ ../../packages/dask-task-models-library simcore-models-library @ ../../packages/models-library pytest-simcore @ ../../packages/pytest-simcore simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/clusters-keeper/requirements/dev.txt b/services/clusters-keeper/requirements/dev.txt index faf4378c83db..004ee6c6241c 100644 --- a/services/clusters-keeper/requirements/dev.txt +++ b/services/clusters-keeper/requirements/dev.txt @@ -18,6 +18,7 @@ --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] --editable ../../packages/settings-library +--editable ../../packages/dask-task-models-library # installs current package --editable . From 4ec4d40e72eb7d62f23421f7496de823c686f0e9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 16:22:43 +0200 Subject: [PATCH 38/46] mypy --- .../src/simcore_service_director_v2/modules/dask_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index a07e83aa52e2..7468f4bd63ad 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -198,7 +198,7 @@ def _comp_sidecar_fct( ) -> TaskOutputData: """This function is serialized by the Dask client and sent over to the Dask sidecar(s) Therefore, (screaming here) DO NOT MOVE THAT IMPORT ANYWHERE ELSE EVER!!""" - from simcore_service_dask_sidecar.worker import ( # type: ignore[import-untyped] # this runs inside the dask-sidecar + from simcore_service_dask_sidecar.worker import ( # type: ignore[import-not-found] # this runs inside the dask-sidecar run_computational_sidecar, ) From c8bec8fcb4ae00b699f21c439c934e0e1688120c Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 16:37:55 +0200 Subject: [PATCH 39/46] fixed tests --- .../modules/comp_scheduler/_scheduler_dask.py | 5 +- .../comp_scheduler/test_scheduler_dask.py | 96 ++++++++++--------- 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py index 0c7cbd6523eb..c3c21fa0b8fa 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py @@ -23,6 +23,9 @@ from models_library.users import UserID from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from servicelib.logging_utils import log_catch +from simcore_service_director_v2.modules.comp_scheduler._utils import ( + WAITING_FOR_START_STATES, +) from ...core.errors import ( ComputationalBackendNotConnectedError, @@ -381,7 +384,7 @@ async def _task_progress_change_handler( node_id = task_progress_event.task_owner.node_id comp_tasks_repo = CompTasksRepository(self.db_engine) task = await comp_tasks_repo.get_task(project_id, node_id) - if task.progress is None: + if task.state in WAITING_FOR_START_STATES: task.state = RunningState.STARTED task.progress = task_progress_event.progress run = await CompRunsRepository(self.db_engine).get(user_id, project_id) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_scheduler_dask.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_scheduler_dask.py index 80666d7f6f03..717bc3381caa 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_scheduler_dask.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_scheduler_dask.py @@ -60,7 +60,6 @@ from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB from simcore_service_director_v2.models.comp_runs import CompRunsAtDB, RunMetadataDict from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB, Image -from simcore_service_director_v2.models.dask_subsystem import DaskClientTaskState from simcore_service_director_v2.modules.comp_scheduler._manager import ( run_new_pipeline, stop_pipeline, @@ -206,8 +205,8 @@ async def _assert_publish_in_dask_backend( for p in expected_pending_tasks: published_tasks.remove(p) - async def _return_tasks_pending(job_ids: list[str]) -> list[DaskClientTaskState]: - return [DaskClientTaskState.PENDING for job_id in job_ids] + async def _return_tasks_pending(job_ids: list[str]) -> list[RunningState]: + return [RunningState.PENDING for job_id in job_ids] mocked_dask_client.get_tasks_status.side_effect = _return_tasks_pending assert published_project.project.prj_owner @@ -445,17 +444,16 @@ async def test_proper_pipeline_is_scheduled( # noqa: PLR0915 ) # ------------------------------------------------------------------------------- - # 2.1. the dask-worker might be taking the task, until we get a progress we do not know - # whether it effectively started or it is still queued in the worker process + # 2.1. the dask-worker takes the task exp_started_task = expected_pending_tasks[0] expected_pending_tasks.remove(exp_started_task) - async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskState]: + async def _return_1st_task_running(job_ids: list[str]) -> list[RunningState]: return [ ( - DaskClientTaskState.PENDING_OR_STARTED + RunningState.STARTED if job_id == exp_started_task.job_id - else DaskClientTaskState.PENDING + else RunningState.PENDING ) for job_id in job_ids ] @@ -469,7 +467,7 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta await assert_comp_runs( sqlalchemy_async_engine, expected_total=1, - expected_state=RunningState.PENDING, + expected_state=RunningState.STARTED, where_statement=and_( comp_runs.c.user_id == published_project.project.prj_owner, comp_runs.c.project_uuid == f"{published_project.project.uuid}", @@ -478,8 +476,14 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta await assert_comp_tasks( sqlalchemy_async_engine, project_uuid=published_project.project.uuid, - task_ids=[exp_started_task.node_id] - + [p.node_id for p in expected_pending_tasks], + task_ids=[exp_started_task.node_id], + expected_state=RunningState.STARTED, + expected_progress=None, + ) + await assert_comp_tasks( + sqlalchemy_async_engine, + project_uuid=published_project.project.uuid, + task_ids=[p.node_id for p in expected_pending_tasks], expected_state=RunningState.PENDING, expected_progress=None, ) @@ -572,12 +576,12 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta # ------------------------------------------------------------------------------- # 4. the dask-worker completed the task successfully - async def _return_1st_task_success(job_ids: list[str]) -> list[DaskClientTaskState]: + async def _return_1st_task_success(job_ids: list[str]) -> list[RunningState]: return [ ( - DaskClientTaskState.SUCCESS + RunningState.SUCCESS if job_id == exp_started_task.job_id - else DaskClientTaskState.PENDING + else RunningState.PENDING ) for job_id in job_ids ] @@ -679,12 +683,12 @@ async def _return_random_task_result(job_id) -> TaskOutputData: # 6. the dask-worker starts processing a task exp_started_task = next_pending_task - async def _return_2nd_task_running(job_ids: list[str]) -> list[DaskClientTaskState]: + async def _return_2nd_task_running(job_ids: list[str]) -> list[RunningState]: return [ ( - DaskClientTaskState.PENDING_OR_STARTED + RunningState.STARTED if job_id == exp_started_task.job_id - else DaskClientTaskState.PENDING + else RunningState.PENDING ) for job_id in job_ids ] @@ -743,12 +747,12 @@ async def _return_2nd_task_running(job_ids: list[str]) -> list[DaskClientTaskSta # ------------------------------------------------------------------------------- # 7. the task fails - async def _return_2nd_task_failed(job_ids: list[str]) -> list[DaskClientTaskState]: + async def _return_2nd_task_failed(job_ids: list[str]) -> list[RunningState]: return [ ( - DaskClientTaskState.ERRED + RunningState.FAILED if job_id == exp_started_task.job_id - else DaskClientTaskState.PENDING + else RunningState.PENDING ) for job_id in job_ids ] @@ -805,12 +809,12 @@ async def _return_2nd_task_failed(job_ids: list[str]) -> list[DaskClientTaskStat # 8. the last task shall succeed exp_started_task = expected_pending_tasks[0] - async def _return_3rd_task_success(job_ids: list[str]) -> list[DaskClientTaskState]: + async def _return_3rd_task_success(job_ids: list[str]) -> list[RunningState]: return [ ( - DaskClientTaskState.SUCCESS + RunningState.SUCCESS if job_id == exp_started_task.job_id - else DaskClientTaskState.PENDING + else RunningState.PENDING ) for job_id in job_ids ] @@ -917,12 +921,12 @@ async def with_started_project( exp_started_task = expected_pending_tasks[0] expected_pending_tasks.remove(exp_started_task) - async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskState]: + async def _return_1st_task_running(job_ids: list[str]) -> list[RunningState]: return [ ( - DaskClientTaskState.PENDING_OR_STARTED + RunningState.STARTED if job_id == exp_started_task.job_id - else DaskClientTaskState.PENDING + else RunningState.PENDING ) for job_id in job_ids ] @@ -939,7 +943,7 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta await assert_comp_runs( sqlalchemy_async_engine, expected_total=1, - expected_state=RunningState.PENDING, + expected_state=RunningState.STARTED, where_statement=and_( comp_runs.c.user_id == published_project.project.prj_owner, comp_runs.c.project_uuid == f"{published_project.project.uuid}", @@ -948,8 +952,14 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta await assert_comp_tasks( sqlalchemy_async_engine, project_uuid=published_project.project.uuid, - task_ids=[exp_started_task.node_id] - + [p.node_id for p in expected_pending_tasks], + task_ids=[exp_started_task.node_id], + expected_state=RunningState.STARTED, + expected_progress=None, + ) + await assert_comp_tasks( + sqlalchemy_async_engine, + project_uuid=published_project.project.uuid, + task_ids=[p.node_id for p in expected_pending_tasks], expected_state=RunningState.PENDING, expected_progress=None, ) @@ -1308,7 +1318,7 @@ async def test_handling_of_disconnected_scheduler_dask( @dataclass(frozen=True, kw_only=True) class RebootState: - dask_task_status: DaskClientTaskState + dask_task_status: RunningState task_result: Exception | TaskOutputData expected_task_state_group1: RunningState expected_task_progress_group1: float @@ -1322,7 +1332,7 @@ class RebootState: [ pytest.param( RebootState( - dask_task_status=DaskClientTaskState.LOST, + dask_task_status=RunningState.UNKNOWN, task_result=ComputationalBackendTaskNotFoundError(job_id="fake_job_id"), expected_task_state_group1=RunningState.FAILED, expected_task_progress_group1=1, @@ -1334,7 +1344,7 @@ class RebootState: ), pytest.param( RebootState( - dask_task_status=DaskClientTaskState.ABORTED, + dask_task_status=RunningState.ABORTED, task_result=TaskCancelledError(job_id="fake_job_id"), expected_task_state_group1=RunningState.ABORTED, expected_task_progress_group1=1, @@ -1346,7 +1356,7 @@ class RebootState: ), pytest.param( RebootState( - dask_task_status=DaskClientTaskState.ERRED, + dask_task_status=RunningState.FAILED, task_result=ValueError("some error during the call"), expected_task_state_group1=RunningState.FAILED, expected_task_progress_group1=1, @@ -1358,7 +1368,7 @@ class RebootState: ), pytest.param( RebootState( - dask_task_status=DaskClientTaskState.PENDING_OR_STARTED, + dask_task_status=RunningState.STARTED, task_result=ComputationalBackendTaskResultsNotReadyError( job_id="fake_job_id" ), @@ -1372,7 +1382,7 @@ class RebootState: ), pytest.param( RebootState( - dask_task_status=DaskClientTaskState.SUCCESS, + dask_task_status=RunningState.SUCCESS, task_result=TaskOutputData.model_validate({"whatever_output": 123}), expected_task_state_group1=RunningState.SUCCESS, expected_task_progress_group1=1, @@ -1399,7 +1409,7 @@ async def test_handling_scheduled_tasks_after_director_reboots( shall continue scheduling correctly. Even though the task might have continued to run in the dask-scheduler.""" - async def mocked_get_tasks_status(job_ids: list[str]) -> list[DaskClientTaskState]: + async def mocked_get_tasks_status(job_ids: list[str]) -> list[RunningState]: return [reboot_state.dask_task_status for j in job_ids] mocked_dask_client.get_tasks_status.side_effect = mocked_get_tasks_status @@ -1514,8 +1524,8 @@ async def test_handling_cancellation_of_jobs_after_reboot( ) # the backend shall report the tasks as running - async def mocked_get_tasks_status(job_ids: list[str]) -> list[DaskClientTaskState]: - return [DaskClientTaskState.PENDING_OR_STARTED for j in job_ids] + async def mocked_get_tasks_status(job_ids: list[str]) -> list[RunningState]: + return [RunningState.STARTED for j in job_ids] mocked_dask_client.get_tasks_status.side_effect = mocked_get_tasks_status # Running the scheduler, should actually cancel the run now @@ -1559,8 +1569,8 @@ async def mocked_get_tasks_status(job_ids: list[str]) -> list[DaskClientTaskStat # the backend shall now report the tasks as aborted async def mocked_get_tasks_status_aborted( job_ids: list[str], - ) -> list[DaskClientTaskState]: - return [DaskClientTaskState.ABORTED for j in job_ids] + ) -> list[RunningState]: + return [RunningState.ABORTED for j in job_ids] mocked_dask_client.get_tasks_status.side_effect = mocked_get_tasks_status_aborted @@ -1641,12 +1651,12 @@ async def test_running_pipeline_triggers_heartbeat( exp_started_task = expected_pending_tasks[0] expected_pending_tasks.remove(exp_started_task) - async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskState]: + async def _return_1st_task_running(job_ids: list[str]) -> list[RunningState]: return [ ( - DaskClientTaskState.PENDING_OR_STARTED + RunningState.STARTED if job_id == exp_started_task.job_id - else DaskClientTaskState.PENDING + else RunningState.PENDING ) for job_id in job_ids ] From 15b135e67c7fab102552cdfbb0606a2217851719 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 16:48:22 +0200 Subject: [PATCH 40/46] pylint --- .../plugins/task_life_cycle_scheduler_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py index 4e0a338abc9b..2590de6ea431 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py @@ -25,7 +25,7 @@ def transition( key: Key, start: TaskStateState, finish: TaskStateState, - *args: Any, # noqa: ARG002 + *args: Any, # noqa: ARG002 #pylint: disable=unused-argument stimulus_id: str, **kwargs: Any, ): From ffbc5a9f597fea9a8516d3bf843aa7216a3df7da Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 16:55:23 +0200 Subject: [PATCH 41/46] fix difference --- .../computational-clusters/autoscaled_monitor/ssh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maintenance/computational-clusters/autoscaled_monitor/ssh.py b/scripts/maintenance/computational-clusters/autoscaled_monitor/ssh.py index e94fed195e42..88486c374286 100644 --- a/scripts/maintenance/computational-clusters/autoscaled_monitor/ssh.py +++ b/scripts/maintenance/computational-clusters/autoscaled_monitor/ssh.py @@ -267,7 +267,7 @@ def _needs_manual_intervention( created_at=containers[0].created_at, needs_manual_intervention=_needs_manual_intervention(containers) and ( - (containers[0].created_at - arrow.utcnow().datetime) + (arrow.utcnow().datetime - containers[0].created_at) > datetime.timedelta(minutes=2) ), containers=[c.name for c in containers], From 1f0ac5fc29966feb665e974d8220df18bfccfd06 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 16:57:43 +0200 Subject: [PATCH 42/46] language --- .../simcore_service_dask_sidecar/rabbitmq_worker_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py index 616a564b3850..ba4936284d7b 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/rabbitmq_worker_plugin.py @@ -60,7 +60,7 @@ async def _() -> None: if threading.current_thread() is not threading.main_thread(): _logger.warning( - "RabbitMQ client plugin setup is not in the main thread! Beware! if in pytest it's ok." + "RabbitMQ client plugin setup is not in the main thread! TIP: if in pytest it's ok." ) with log_context( @@ -98,7 +98,7 @@ async def _() -> None: ) else: _logger.warning( - "RabbitMQ client plugin setup is not the main thread! Beware! if in pytest it's ok." + "RabbitMQ client plugin setup is not the main thread! TIP: if in pytest it's ok." ) # Cancel the message processor task From 23c1fcb50af3ddc2e6adb6ee7eda4b4c39ec79a8 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 17:15:16 +0200 Subject: [PATCH 43/46] pylint --- .../plugins/task_life_cycle_scheduler_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py index 2590de6ea431..6d943b841dda 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py @@ -25,7 +25,7 @@ def transition( key: Key, start: TaskStateState, finish: TaskStateState, - *args: Any, # noqa: ARG002 #pylint: disable=unused-argument + *args: Any, # pylint: disable=unused-argument # noqa: ARG002 stimulus_id: str, **kwargs: Any, ): From 3230606d73bec9a7c365e150174adba98aff720b Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 17:27:59 +0200 Subject: [PATCH 44/46] pylint --- .../plugins/task_life_cycle_scheduler_plugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py index 6d943b841dda..69fbc35e21ec 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/plugins/task_life_cycle_scheduler_plugin.py @@ -1,3 +1,4 @@ +# pylint: disable=unused-argument import logging from typing import Any @@ -25,7 +26,7 @@ def transition( key: Key, start: TaskStateState, finish: TaskStateState, - *args: Any, # pylint: disable=unused-argument # noqa: ARG002 + *args: Any, # noqa: ARG002 stimulus_id: str, **kwargs: Any, ): From ab92c7041979b73bf28939dd63c32d8d9c79fea6 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 May 2025 17:34:13 +0200 Subject: [PATCH 45/46] timeout increase --- .../api/routes/computations.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py index 8a7809ab7d65..9f096911030a 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py @@ -17,6 +17,7 @@ import contextlib import logging +from datetime import timedelta from typing import Annotated, Any, Final import networkx as nx @@ -84,7 +85,7 @@ from ..dependencies.rut_client import get_rut_client from .computations_tasks import analyze_pipeline -_PIPELINE_ABORT_TIMEOUT_S: Final[int] = 10 +_PIPELINE_ABORT_TIMEOUT_S: Final[timedelta] = timedelta(seconds=30) _logger = logging.getLogger(__name__) @@ -121,7 +122,7 @@ async def _check_pipeline_startable( ): raise HTTPException( status_code=status.HTTP_406_NOT_ACCEPTABLE, - detail=f"Project {computation.project_id} cannot run since it contains deprecated tasks {jsonable_encoder( deprecated_tasks)}", + detail=f"Project {computation.project_id} cannot run since it contains deprecated tasks {jsonable_encoder(deprecated_tasks)}", ) @@ -624,7 +625,7 @@ def return_last_value(retry_state: Any) -> Any: return retry_state.outcome.result() @retry( - stop=stop_after_delay(_PIPELINE_ABORT_TIMEOUT_S), + stop=stop_after_delay(_PIPELINE_ABORT_TIMEOUT_S.total_seconds()), wait=wait_random(0, 2), retry_error_callback=return_last_value, retry=retry_if_result(lambda result: result is False), @@ -643,7 +644,7 @@ async def check_pipeline_stopped() -> bool: # wait for the pipeline to be stopped if not await check_pipeline_stopped(): _logger.error( - "pipeline %s could not be stopped properly after %ss", + "pipeline %s could not be stopped properly after %s", project_id, _PIPELINE_ABORT_TIMEOUT_S, ) From 1567bdd7802f3a0308b57d86dbe06033625a40dd Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 20 May 2025 09:09:24 +0200 Subject: [PATCH 46/46] revert changes --- services/director-v2/requirements/_test.in | 6 +++++- services/director-v2/requirements/_test.txt | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/services/director-v2/requirements/_test.in b/services/director-v2/requirements/_test.in index 2fb831189bae..3633a09b7044 100644 --- a/services/director-v2/requirements/_test.in +++ b/services/director-v2/requirements/_test.in @@ -19,7 +19,11 @@ docker Faker flaky pytest -pytest-asyncio +# ---- +# Overcomes "Known issues" in https://github.com/pytest-dev/pytest-asyncio/releases/tag/v0.23.8 +# IMPORTANT: This constraint can be removed when `test_pytest_asyncio_known_issue` passes with the new update of pytest-asyncio +pytest-asyncio<0.23 +# ---- pytest-cov pytest-docker pytest-icdiff diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index f28a0daa1544..08c3214f5da2 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -232,7 +232,7 @@ pytest==8.3.5 # pytest-icdiff # pytest-mock # pytest-xdist -pytest-asyncio==0.23.8 +pytest-asyncio==0.21.2 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in