From 07dbc67a8ec5374b7cc239f918bfadb9e27f9f6f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 5 Jun 2025 15:35:53 +0200 Subject: [PATCH 01/46] changing visibility --- api/specs/web-server/_long_running_tasks.py | 2 +- api/specs/web-server/_long_running_tasks_legacy.py | 2 +- .../aiohttp/long_running_tasks/_dependencies.py | 2 +- .../aiohttp/long_running_tasks/_error_handlers.py | 2 +- .../aiohttp/long_running_tasks/_routes.py | 6 +++--- .../aiohttp/long_running_tasks/_server.py | 14 +++++++------- .../aiohttp/long_running_tasks/client.py | 2 +- .../aiohttp/long_running_tasks/server.py | 6 +++--- .../fastapi/long_running_tasks/_client.py | 4 ++-- .../fastapi/long_running_tasks/_context_manager.py | 4 ++-- .../fastapi/long_running_tasks/_dependencies.py | 2 +- .../fastapi/long_running_tasks/_error_handlers.py | 2 +- .../fastapi/long_running_tasks/_routes.py | 6 +++--- .../fastapi/long_running_tasks/_server.py | 6 +++--- .../fastapi/long_running_tasks/client.py | 4 ++-- .../fastapi/long_running_tasks/server.py | 4 ++-- .../long_running_tasks/{_errors.py => errors.py} | 0 .../long_running_tasks/{_models.py => models.py} | 0 .../long_running_tasks/{_task.py => task.py} | 6 +++--- .../tests/aiohttp/long_running_tasks/conftest.py | 2 +- .../long_running_tasks/test_long_running_tasks.py | 2 +- .../test_long_running_tasks_with_task_context.py | 2 +- .../long_running_tasks/test_long_running_tasks.py | 4 ++-- .../test_long_running_tasks_context_manager.py | 2 +- .../test_long_running_tasks_models.py | 2 +- .../test_long_running_tasks_task.py | 6 +++--- .../director-v2/tests/unit/with_dbs/test_cli.py | 2 +- 27 files changed, 48 insertions(+), 48 deletions(-) rename packages/service-library/src/servicelib/long_running_tasks/{_errors.py => errors.py} (100%) rename packages/service-library/src/servicelib/long_running_tasks/{_models.py => models.py} (100%) rename packages/service-library/src/servicelib/long_running_tasks/{_task.py => task.py} (99%) diff --git a/api/specs/web-server/_long_running_tasks.py b/api/specs/web-server/_long_running_tasks.py index f204c1de5b4..fc99db45b89 100644 --- a/api/specs/web-server/_long_running_tasks.py +++ b/api/specs/web-server/_long_running_tasks.py @@ -10,7 +10,7 @@ from models_library.generics import Envelope from models_library.rest_error import EnvelopedError from servicelib.aiohttp.long_running_tasks._routes import _PathParam -from servicelib.long_running_tasks._models import TaskGet, TaskStatus +from servicelib.long_running_tasks.models import TaskGet, TaskStatus from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.tasks._exception_handlers import ( _TO_HTTP_ERROR_MAP as export_data_http_error_map, diff --git a/api/specs/web-server/_long_running_tasks_legacy.py b/api/specs/web-server/_long_running_tasks_legacy.py index fcbe3508c4d..89c85b0ff93 100644 --- a/api/specs/web-server/_long_running_tasks_legacy.py +++ b/api/specs/web-server/_long_running_tasks_legacy.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, status from models_library.generics import Envelope from servicelib.aiohttp.long_running_tasks._routes import _PathParam -from servicelib.long_running_tasks._models import TaskGet, TaskStatus +from servicelib.long_running_tasks.models import TaskGet, TaskStatus from simcore_service_webserver._meta import API_VTAG router = APIRouter( diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py index b38004b3200..4a8bc455cd8 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py @@ -2,7 +2,7 @@ from aiohttp import web -from ...long_running_tasks._task import TasksManager +from ...long_running_tasks.task import TasksManager from ._constants import ( APP_LONG_RUNNING_TASKS_MANAGER_KEY, RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_error_handlers.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_error_handlers.py index 4534d7c951c..8a679c70b7d 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_error_handlers.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_error_handlers.py @@ -3,7 +3,7 @@ from aiohttp import web from common_library.json_serialization import json_dumps -from ...long_running_tasks._errors import ( +from ...long_running_tasks.errors import ( TaskCancelledError, TaskNotCompletedError, TaskNotFoundError, diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 1906c0bc93f..2603b229f68 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -6,9 +6,9 @@ from pydantic import BaseModel from servicelib.aiohttp import status -from ...long_running_tasks._errors import TaskNotCompletedError, TaskNotFoundError -from ...long_running_tasks._models import TaskGet, TaskId, TaskStatus -from ...long_running_tasks._task import TrackedTask +from ...long_running_tasks.errors import TaskNotCompletedError, TaskNotFoundError +from ...long_running_tasks.models import TaskGet, TaskId, TaskStatus +from ...long_running_tasks.task import TrackedTask from ..requests_validation import parse_request_path_parameters_as from ._dependencies import get_task_context, get_tasks_manager diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index d0c96699462..40777376815 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -9,8 +9,8 @@ from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter from ...aiohttp import status -from ...long_running_tasks._models import TaskGet -from ...long_running_tasks._task import ( +from ...long_running_tasks.models import TaskGet +from ...long_running_tasks.task import ( TaskContext, TaskProtocol, TasksManager, @@ -136,11 +136,11 @@ def setup( async def on_cleanup_ctx(app: web.Application) -> AsyncGenerator[None, None]: # add components to state - app[ - APP_LONG_RUNNING_TASKS_MANAGER_KEY - ] = long_running_task_manager = TasksManager( - stale_task_check_interval_s=stale_task_check_interval_s, - stale_task_detect_timeout_s=stale_task_detect_timeout_s, + app[APP_LONG_RUNNING_TASKS_MANAGER_KEY] = long_running_task_manager = ( + TasksManager( + stale_task_check_interval_s=stale_task_check_interval_s, + stale_task_detect_timeout_s=stale_task_detect_timeout_s, + ) ) # add error handlers diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py index e29fabc87fe..cdf9669cd10 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py @@ -13,7 +13,7 @@ from yarl import URL from ...long_running_tasks._constants import DEFAULT_POLL_INTERVAL_S, HOUR -from ...long_running_tasks._models import LRTask, RequestBody +from ...long_running_tasks.models import LRTask, RequestBody from ...rest_responses import unwrap_envelope_if_required from .. import status from .server import TaskGet, TaskId, TaskProgress, TaskStatus diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py index 55d1295c197..db574df7f3f 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py @@ -6,9 +6,9 @@ running task. """ -from ...long_running_tasks._errors import TaskAlreadyRunningError, TaskCancelledError -from ...long_running_tasks._models import ProgressMessage, ProgressPercent -from ...long_running_tasks._task import ( +from ...long_running_tasks.errors import TaskAlreadyRunningError, TaskCancelledError +from ...long_running_tasks.models import ProgressMessage, ProgressPercent +from ...long_running_tasks.task import ( TaskId, TaskProgress, TaskProtocol, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py index 4593bbf7b01..e3aa8902f65 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py @@ -14,8 +14,8 @@ from tenacity.stop import stop_after_attempt from tenacity.wait import wait_exponential -from ...long_running_tasks._errors import GenericClientError -from ...long_running_tasks._models import ClientConfiguration, TaskId, TaskStatus +from ...long_running_tasks.errors import GenericClientError +from ...long_running_tasks.models import ClientConfiguration, TaskId, TaskStatus DEFAULT_HTTP_REQUESTS_TIMEOUT: Final[PositiveFloat] = 15 diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py index c16fadd8be2..3cc1b65c8f6 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py @@ -6,8 +6,8 @@ from pydantic import PositiveFloat -from ...long_running_tasks._errors import TaskClientTimeoutError -from ...long_running_tasks._models import ( +from ...long_running_tasks.errors import TaskClientTimeoutError +from ...long_running_tasks.models import ( ProgressCallback, ProgressMessage, ProgressPercent, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_dependencies.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_dependencies.py index 937ddcf33d1..e2c2fdc4b00 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_dependencies.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_dependencies.py @@ -1,6 +1,6 @@ from fastapi import Request -from ...long_running_tasks._task import TasksManager +from ...long_running_tasks.task import TasksManager def get_tasks_manager(request: Request) -> TasksManager: diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_error_handlers.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_error_handlers.py index e5f1ef7d9ee..b39eff575bc 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_error_handlers.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_error_handlers.py @@ -5,7 +5,7 @@ from starlette.requests import Request from starlette.responses import JSONResponse -from ...long_running_tasks._errors import ( +from ...long_running_tasks.errors import ( BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index b56ba3d21dd..fa11e506012 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -2,9 +2,9 @@ from fastapi import APIRouter, Depends, Request, status -from ...long_running_tasks._errors import TaskNotCompletedError, TaskNotFoundError -from ...long_running_tasks._models import TaskGet, TaskId, TaskResult, TaskStatus -from ...long_running_tasks._task import TasksManager +from ...long_running_tasks.errors import TaskNotCompletedError, TaskNotFoundError +from ...long_running_tasks.models import TaskGet, TaskId, TaskResult, TaskStatus +from ...long_running_tasks.task import TasksManager from ..requests_decorators import cancel_on_disconnect from ._dependencies import get_tasks_manager diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index e8306b6d187..26c26d10bc3 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, FastAPI from pydantic import PositiveFloat -from ...long_running_tasks._errors import BaseLongRunningError -from ...long_running_tasks._task import TasksManager +from ...long_running_tasks.errors import BaseLongRunningError +from ...long_running_tasks.task import TasksManager from ._error_handlers import base_long_running_error_handler from ._routes import router @@ -50,4 +50,4 @@ async def on_shutdown() -> None: # add error handlers # NOTE: Exception handler can not be added during the on_startup script, otherwise not working correctly - app.add_exception_handler(BaseLongRunningError, base_long_running_error_handler) # type: ignore[arg-type] + app.add_exception_handler(BaseLongRunningError, base_long_running_error_handler) # type: ignore[arg-type] diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py index 62b72256000..32eb9c6482a 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py @@ -27,7 +27,7 @@ from yarl import URL from ...long_running_tasks._constants import DEFAULT_POLL_INTERVAL_S, HOUR -from ...long_running_tasks._models import ( +from ...long_running_tasks.models import ( ClientConfiguration, LRTask, ProgressCallback, @@ -35,7 +35,7 @@ ProgressPercent, RequestBody, ) -from ...long_running_tasks._task import TaskId +from ...long_running_tasks.task import TaskId from ...rest_responses import unwrap_envelope_if_required from ._client import DEFAULT_HTTP_REQUESTS_TIMEOUT, Client, setup from ._context_manager import periodic_task_result diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py index b9a29d1d90a..e1decb5200f 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py @@ -8,8 +8,8 @@ from models_library.api_schemas_long_running_tasks.tasks import TaskResult -from ...long_running_tasks._errors import TaskAlreadyRunningError, TaskCancelledError -from ...long_running_tasks._task import ( +from ...long_running_tasks.errors import TaskAlreadyRunningError, TaskCancelledError +from ...long_running_tasks.task import ( TaskId, TaskProgress, TasksManager, diff --git a/packages/service-library/src/servicelib/long_running_tasks/_errors.py b/packages/service-library/src/servicelib/long_running_tasks/errors.py similarity index 100% rename from packages/service-library/src/servicelib/long_running_tasks/_errors.py rename to packages/service-library/src/servicelib/long_running_tasks/errors.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/_models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py similarity index 100% rename from packages/service-library/src/servicelib/long_running_tasks/_models.py rename to packages/service-library/src/servicelib/long_running_tasks/models.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/_task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py similarity index 99% rename from packages/service-library/src/servicelib/long_running_tasks/_task.py rename to packages/service-library/src/servicelib/long_running_tasks/task.py index b1b0bedfcc0..5e02b1adbec 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -15,14 +15,14 @@ ) from pydantic import PositiveFloat -from ._errors import ( +from .errors import ( TaskAlreadyRunningError, TaskCancelledError, TaskExceptionError, TaskNotCompletedError, TaskNotFoundError, ) -from ._models import TaskId, TaskName, TaskStatus, TrackedTask +from .models import TaskId, TaskName, TaskStatus, TrackedTask logger = logging.getLogger(__name__) @@ -262,7 +262,7 @@ async def _cancel_asyncio_task( await asyncio.wait_for( _await_task(task), timeout=self._cancel_task_timeout_s ) - except asyncio.TimeoutError: + except TimeoutError: logger.warning( "Timed out while awaiting for cancellation of '%s'", reference ) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py index 8fe29473cfc..1ae88d8b724 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py @@ -14,7 +14,7 @@ from servicelib.aiohttp import long_running_tasks, status from servicelib.aiohttp.long_running_tasks.server import TaskId from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as -from servicelib.long_running_tasks._task import TaskContext +from servicelib.long_running_tasks.task import TaskContext from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index 71d5501b2ed..7d5f24b7a0e 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -22,7 +22,7 @@ from servicelib.aiohttp import long_running_tasks, status from servicelib.aiohttp.long_running_tasks.server import TaskGet, TaskId from servicelib.aiohttp.rest_middlewares import append_rest_middlewares -from servicelib.long_running_tasks._task import TaskContext +from servicelib.long_running_tasks.task import TaskContext from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py index 0b37c941669..16ea2831021 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py @@ -26,7 +26,7 @@ from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as from servicelib.aiohttp.rest_middlewares import append_rest_middlewares from servicelib.aiohttp.typing_extension import Handler -from servicelib.long_running_tasks._task import TaskContext +from servicelib.long_running_tasks.task import TaskContext # WITH TASK CONTEXT # NOTE: as the long running task framework may be used in any number of services diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index 84146c6b0dc..3c7c06db472 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -20,8 +20,8 @@ from httpx import AsyncClient from pydantic import TypeAdapter from servicelib.fastapi import long_running_tasks -from servicelib.long_running_tasks._models import TaskGet, TaskId -from servicelib.long_running_tasks._task import TaskContext +from servicelib.long_running_tasks.models import TaskGet, TaskId +from servicelib.long_running_tasks.task import TaskContext from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index b0db697a6ad..b1739e761c7 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -28,7 +28,7 @@ from servicelib.fastapi.long_running_tasks.server import ( start_task, ) -from servicelib.long_running_tasks._errors import ( +from servicelib.long_running_tasks.errors import ( TaskClientTimeoutError, ) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_models.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_models.py index f21417da788..a92765eb010 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_models.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_models.py @@ -1,4 +1,4 @@ -from servicelib.long_running_tasks._models import TaskProgress +from servicelib.long_running_tasks.models import TaskProgress def test_progress_has_no_more_than_3_digits(): diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 6d3b9c837f2..63be0eb1262 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -12,18 +12,18 @@ import pytest from faker import Faker -from servicelib.long_running_tasks._errors import ( +from servicelib.long_running_tasks.errors import ( TaskAlreadyRunningError, TaskCancelledError, TaskNotCompletedError, TaskNotFoundError, ) -from servicelib.long_running_tasks._models import ( +from servicelib.long_running_tasks.models import ( ProgressPercent, TaskProgress, TaskStatus, ) -from servicelib.long_running_tasks._task import TasksManager, start_task +from servicelib.long_running_tasks.task import TasksManager, start_task from tenacity import TryAgain from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type diff --git a/services/director-v2/tests/unit/with_dbs/test_cli.py b/services/director-v2/tests/unit/with_dbs/test_cli.py index 813bd93aa07..2f7706cbda1 100644 --- a/services/director-v2/tests/unit/with_dbs/test_cli.py +++ b/services/director-v2/tests/unit/with_dbs/test_cli.py @@ -22,7 +22,7 @@ from models_library.projects_nodes_io import NodeID from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict -from servicelib.long_running_tasks._models import ProgressCallback +from servicelib.long_running_tasks.models import ProgressCallback from simcore_service_director_v2.cli import DEFAULT_NODE_SAVE_ATTEMPTS, main from simcore_service_director_v2.cli._close_and_save_service import ( ThinDV2LocalhostClient, From 4c689a2f021902cccbe7013e479e5166d3ba1a9a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 5 Jun 2025 15:41:15 +0200 Subject: [PATCH 02/46] fixed imports --- .../aiohttp/long_running_tasks/client.py | 13 ++++++++----- .../aiohttp/long_running_tasks/server.py | 18 ------------------ .../aiohttp/long_running_tasks/conftest.py | 4 ++-- .../test_long_running_tasks.py | 2 +- ...est_long_running_tasks_with_task_context.py | 2 +- .../test_long_running_tasks.py | 4 ++-- 6 files changed, 14 insertions(+), 29 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py index cdf9669cd10..81c48ed017b 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py @@ -13,10 +13,16 @@ from yarl import URL from ...long_running_tasks._constants import DEFAULT_POLL_INTERVAL_S, HOUR -from ...long_running_tasks.models import LRTask, RequestBody +from ...long_running_tasks.models import ( + LRTask, + RequestBody, + TaskGet, + TaskId, + TaskProgress, + TaskStatus, +) from ...rest_responses import unwrap_envelope_if_required from .. import status -from .server import TaskGet, TaskId, TaskProgress, TaskStatus _logger = logging.getLogger(__name__) @@ -123,6 +129,3 @@ async def long_running_task_request( if task: await _abort_task(session, URL(task.abort_href)) raise - - -__all__: tuple[str, ...] = ("LRTask",) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py index db574df7f3f..cb897e3f313 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py @@ -6,15 +6,6 @@ running task. """ -from ...long_running_tasks.errors import TaskAlreadyRunningError, TaskCancelledError -from ...long_running_tasks.models import ProgressMessage, ProgressPercent -from ...long_running_tasks.task import ( - TaskId, - TaskProgress, - TaskProtocol, - TasksManager, - TaskStatus, -) from ._dependencies import ( create_task_name_from_request, get_task_context, @@ -27,18 +18,9 @@ "create_task_name_from_request", "get_task_context", "get_tasks_manager", - "ProgressMessage", - "ProgressPercent", "setup", "start_long_running_task", - "TaskAlreadyRunningError", - "TaskCancelledError", - "TaskId", "TaskGet", - "TasksManager", - "TaskProgress", - "TaskProtocol", - "TaskStatus", ) # nopycln: file diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py index 1ae88d8b724..543869bbc04 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py @@ -12,8 +12,8 @@ from pydantic import BaseModel, TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import long_running_tasks, status -from servicelib.aiohttp.long_running_tasks.server import TaskId from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as +from servicelib.long_running_tasks.models import TaskId, TaskProgress from servicelib.long_running_tasks.task import TaskContext from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -22,7 +22,7 @@ async def _string_list_task( - task_progress: long_running_tasks.server.TaskProgress, + task_progress: TaskProgress, num_strings: int, sleep_time: float, fail: bool, diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index 7d5f24b7a0e..b2ebd569095 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -20,8 +20,8 @@ from pydantic import TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import long_running_tasks, status -from servicelib.aiohttp.long_running_tasks.server import TaskGet, TaskId from servicelib.aiohttp.rest_middlewares import append_rest_middlewares +from servicelib.long_running_tasks.models import TaskGet, TaskId from servicelib.long_running_tasks.task import TaskContext from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py index 16ea2831021..20f8dbf657f 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py @@ -22,10 +22,10 @@ from servicelib.aiohttp.long_running_tasks._server import ( RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, ) -from servicelib.aiohttp.long_running_tasks.server import TaskGet, TaskId from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as from servicelib.aiohttp.rest_middlewares import append_rest_middlewares from servicelib.aiohttp.typing_extension import Handler +from servicelib.long_running_tasks.models import TaskGet, TaskId from servicelib.long_running_tasks.task import TaskContext # WITH TASK CONTEXT diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index 3c7c06db472..aee52da6fc8 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -20,7 +20,7 @@ from httpx import AsyncClient from pydantic import TypeAdapter from servicelib.fastapi import long_running_tasks -from servicelib.long_running_tasks.models import TaskGet, TaskId +from servicelib.long_running_tasks.models import TaskGet, TaskId, TaskProgress from servicelib.long_running_tasks.task import TaskContext from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -32,7 +32,7 @@ async def _string_list_task( - task_progress: long_running_tasks.server.TaskProgress, + task_progress: TaskProgress, num_strings: int, sleep_time: float, fail: bool, From 7977cd7b8802e718c45b49d7b45c9860db766e2f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 5 Jun 2025 15:43:14 +0200 Subject: [PATCH 03/46] fixed imports --- .../tests/aiohttp/long_running_tasks/conftest.py | 11 ++++++++--- .../long_running_tasks/test_long_running_tasks.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py index 543869bbc04..4f39c80be08 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py @@ -13,7 +13,12 @@ from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import long_running_tasks, status from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as -from servicelib.long_running_tasks.models import TaskId, TaskProgress +from servicelib.long_running_tasks.models import ( + TaskGet, + TaskId, + TaskProgress, + TaskStatus, +) from servicelib.long_running_tasks.task import TaskContext from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -93,7 +98,7 @@ async def _caller(client: TestClient, **query_kwargs) -> TaskId: data, error = await assert_status(resp, status.HTTP_202_ACCEPTED) assert data assert not error - task_get = TypeAdapter(long_running_tasks.server.TaskGet).validate_python(data) + task_get = TypeAdapter(TaskGet).validate_python(data) return task_get.task_id return _caller @@ -123,7 +128,7 @@ async def _waiter( data, error = await assert_status(result, status.HTTP_200_OK) assert data assert not error - task_status = long_running_tasks.server.TaskStatus.model_validate(data) + task_status = TaskStatus.model_validate(data) assert task_status assert task_status.done diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index b2ebd569095..ad924ec8e73 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -21,7 +21,7 @@ from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import long_running_tasks, status from servicelib.aiohttp.rest_middlewares import append_rest_middlewares -from servicelib.long_running_tasks.models import TaskGet, TaskId +from servicelib.long_running_tasks.models import TaskGet, TaskId, TaskStatus from servicelib.long_running_tasks.task import TaskContext from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -70,7 +70,7 @@ async def test_workflow( data, error = await assert_status(result, status.HTTP_200_OK) assert data assert not error - task_status = long_running_tasks.server.TaskStatus.model_validate(data) + task_status = TaskStatus.model_validate(data) assert task_status progress_updates.append( (task_status.task_progress.message, task_status.task_progress.percent) From d7f04d1a097f396bc54db26011b5f10cb016564d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 5 Jun 2025 15:47:01 +0200 Subject: [PATCH 04/46] removed unrequired --- .../src/servicelib/aiohttp/long_running_tasks/server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py index cb897e3f313..218fb4640f7 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py @@ -11,7 +11,6 @@ get_task_context, get_tasks_manager, ) -from ._routes import TaskGet from ._server import setup, start_long_running_task __all__: tuple[str, ...] = ( @@ -20,7 +19,6 @@ "get_tasks_manager", "setup", "start_long_running_task", - "TaskGet", ) # nopycln: file From f50e77d40ea5fc6f7eac200da026829b104ca0e5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 5 Jun 2025 16:00:12 +0200 Subject: [PATCH 05/46] refactor imports --- .../fastapi/long_running_tasks/_client.py | 4 ++-- .../fastapi/long_running_tasks/client.py | 23 ++++--------------- ...test_long_running_tasks_context_manager.py | 8 ++----- .../api/routes/dynamic_scheduler.py | 8 +++---- .../cli/_close_and_save_service.py | 3 +-- .../dynamic_sidecar/api_client/_public.py | 8 ++++--- .../modules/dynamic_sidecar/scheduler/_abc.py | 2 +- .../scheduler/_core/_events_utils.py | 2 +- .../scheduler/_core/_scheduler.py | 20 ++++++++-------- .../scheduler/_core/_scheduler_utils.py | 2 +- .../dynamic_sidecar/scheduler/_task.py | 2 +- ...t_dynamic_sidecar_nodeports_integration.py | 3 +-- 12 files changed, 33 insertions(+), 52 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py index e3aa8902f65..c2b93594ce1 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py @@ -17,7 +17,7 @@ from ...long_running_tasks.errors import GenericClientError from ...long_running_tasks.models import ClientConfiguration, TaskId, TaskStatus -DEFAULT_HTTP_REQUESTS_TIMEOUT: Final[PositiveFloat] = 15 +_DEFAULT_HTTP_REQUESTS_TIMEOUT: Final[PositiveFloat] = 15 logger = logging.getLogger(__name__) @@ -207,7 +207,7 @@ def setup( app: FastAPI, *, router_prefix: str = "", - http_requests_timeout: PositiveFloat = DEFAULT_HTTP_REQUESTS_TIMEOUT, + http_requests_timeout: PositiveFloat = _DEFAULT_HTTP_REQUESTS_TIMEOUT, ): """ - `router_prefix` by default it is assumed the server mounts the APIs on diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py index 32eb9c6482a..e0eafea2257 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py @@ -9,12 +9,6 @@ import httpx from fastapi import status -from models_library.api_schemas_long_running_tasks.base import TaskProgress -from models_library.api_schemas_long_running_tasks.tasks import ( - TaskGet, - TaskResult, - TaskStatus, -) from tenacity import ( AsyncRetrying, TryAgain, @@ -28,16 +22,15 @@ from ...long_running_tasks._constants import DEFAULT_POLL_INTERVAL_S, HOUR from ...long_running_tasks.models import ( - ClientConfiguration, LRTask, - ProgressCallback, - ProgressMessage, - ProgressPercent, RequestBody, + TaskGet, + TaskProgress, + TaskStatus, ) from ...long_running_tasks.task import TaskId from ...rest_responses import unwrap_envelope_if_required -from ._client import DEFAULT_HTTP_REQUESTS_TIMEOUT, Client, setup +from ._client import Client, setup from ._context_manager import periodic_task_result _logger = logging.getLogger(__name__) @@ -151,15 +144,7 @@ async def long_running_task_request( __all__: tuple[str, ...] = ( - "DEFAULT_HTTP_REQUESTS_TIMEOUT", "Client", - "ClientConfiguration", - "LRTask", - "ProgressCallback", - "ProgressMessage", - "ProgressPercent", - "TaskId", - "TaskResult", "periodic_task_result", "setup", ) diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index b1739e761c7..eac737268b5 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -11,12 +11,7 @@ from httpx import AsyncClient from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter from servicelib.fastapi.long_running_tasks._context_manager import _ProgressManager -from servicelib.fastapi.long_running_tasks.client import ( - Client, - ProgressMessage, - ProgressPercent, - periodic_task_result, -) +from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result from servicelib.fastapi.long_running_tasks.client import setup as setup_client from servicelib.fastapi.long_running_tasks.server import ( TaskId, @@ -31,6 +26,7 @@ from servicelib.long_running_tasks.errors import ( TaskClientTimeoutError, ) +from servicelib.long_running_tasks.models import ProgressMessage, ProgressPercent TASK_SLEEP_INTERVAL: Final[PositiveFloat] = 0.1 diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py index dadfdc3cdfc..6934ce5056c 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py @@ -4,10 +4,6 @@ from fastapi import APIRouter, Depends, HTTPException, status from models_library.projects_nodes_io import NodeID from pydantic import BaseModel, PositiveInt -from servicelib.fastapi.long_running_tasks.client import ( - ProgressMessage, - ProgressPercent, -) from servicelib.fastapi.long_running_tasks.server import ( TaskAlreadyRunningError, TaskId, @@ -16,6 +12,10 @@ get_tasks_manager, start_task, ) +from servicelib.long_running_tasks.models import ( + ProgressMessage, + ProgressPercent, +) from tenacity import retry from tenacity.before_sleep import before_sleep_log from tenacity.retry import retry_if_result diff --git a/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py b/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py index fb8f70bf62f..bffee5de248 100644 --- a/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py +++ b/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py @@ -16,12 +16,11 @@ from servicelib.fastapi.http_client_thin import UnexpectedStatusError from servicelib.fastapi.long_running_tasks.client import ( Client, - ProgressMessage, - ProgressPercent, TaskId, periodic_task_result, setup, ) +from servicelib.long_running_tasks.models import ProgressMessage, ProgressPercent from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_attempt diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py index 5945e07b8e3..9b96137af3b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py @@ -24,13 +24,15 @@ ) from servicelib.fastapi.long_running_tasks.client import ( Client, - ProgressCallback, - ProgressMessage, - ProgressPercent, TaskId, periodic_task_result, ) from servicelib.logging_utils import log_context, log_decorator +from servicelib.long_running_tasks.models import ( + ProgressCallback, + ProgressMessage, + ProgressPercent, +) from servicelib.utils import logged_gather from ....core.dynamic_services_settings.scheduler import ( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py index fc550e6a74d..869ea3d39f5 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py @@ -15,8 +15,8 @@ from models_library.services_types import ServicePortKey from models_library.users import UserID from models_library.wallets import WalletID -from servicelib.fastapi.long_running_tasks.client import ProgressCallback from servicelib.fastapi.long_running_tasks.server import TaskProgress +from servicelib.long_running_tasks.models import ProgressCallback class SchedulerInternalsInterface(ABC): diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index af3c094a57e..7579d7c1b67 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -20,9 +20,9 @@ from models_library.user_preferences import FrontendUserPreference from models_library.users import UserID from servicelib.fastapi.http_client_thin import BaseHttpClientError -from servicelib.fastapi.long_running_tasks.client import ProgressCallback from servicelib.fastapi.long_running_tasks.server import TaskProgress from servicelib.logging_utils import log_context +from servicelib.long_running_tasks.models import ProgressCallback from servicelib.rabbitmq import RabbitMQClient from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from servicelib.rabbitmq._errors import RemoteMethodNotRegisteredError diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index 6860717238d..d3ef9ffc52d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -43,8 +43,8 @@ from pydantic import NonNegativeFloat from servicelib.async_utils import cancel_wait_task from servicelib.background_task import create_periodic_task -from servicelib.fastapi.long_running_tasks.client import ProgressCallback from servicelib.fastapi.long_running_tasks.server import TaskProgress +from servicelib.long_running_tasks.models import ProgressCallback from servicelib.redis import RedisClientsManager, exclusive from settings_library.redis import RedisDatabase @@ -272,9 +272,9 @@ async def add_service_from_scheduler_data( ) raise DynamicSidecarError(msg=msg) - self._inverse_search_mapping[ - scheduler_data.node_uuid - ] = scheduler_data.service_name + self._inverse_search_mapping[scheduler_data.node_uuid] = ( + scheduler_data.service_name + ) self._to_observe[scheduler_data.service_name] = scheduler_data self._enqueue_observation_from_service_name(scheduler_data.service_name) logger.debug("Added service '%s' to observe", scheduler_data.service_name) @@ -374,9 +374,9 @@ async def mark_service_for_removal( dynamic_scheduler: DynamicServicesSchedulerSettings = ( self.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER ) - self._service_observation_task[ - service_name - ] = self.__create_observation_task(dynamic_scheduler, service_name) + self._service_observation_task[service_name] = ( + self.__create_observation_task(dynamic_scheduler, service_name) + ) logger.debug("Service '%s' marked for removal from scheduler", service_name) @@ -575,9 +575,9 @@ async def _run_trigger_observation_queue_task(self) -> None: if self._service_observation_task.get(service_name) is None: logger.info("Create observation task for service %s", service_name) - self._service_observation_task[ - service_name - ] = self.__create_observation_task(dynamic_scheduler, service_name) + self._service_observation_task[service_name] = ( + self.__create_observation_task(dynamic_scheduler, service_name) + ) logger.info("Scheduler 'trigger observation queue task' was shut down") diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py index 5a4a011a874..3bbd06b7c5c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py @@ -7,7 +7,7 @@ ) from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceBootType, ServiceState -from servicelib.fastapi.long_running_tasks.client import ProgressCallback +from servicelib.long_running_tasks.models import ProgressCallback from .....core.dynamic_services_settings.scheduler import ( DynamicServicesSchedulerSettings, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py index e712958a32a..86592ebcaf9 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py @@ -17,8 +17,8 @@ from models_library.services_types import ServicePortKey from models_library.users import UserID from models_library.wallets import WalletID -from servicelib.fastapi.long_running_tasks.client import ProgressCallback from servicelib.fastapi.long_running_tasks.server import TaskProgress +from servicelib.long_running_tasks.models import ProgressCallback from ....core.dynamic_services_settings.scheduler import ( DynamicServicesSchedulerSettings, diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 30563157d6d..3469914a677 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -52,11 +52,10 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.fastapi.long_running_tasks.client import ( Client, - ProgressMessage, - ProgressPercent, TaskId, periodic_task_result, ) +from servicelib.long_running_tasks.models import ProgressMessage, ProgressPercent from servicelib.progress_bar import ProgressBarData from servicelib.sequences_utils import pairwise from settings_library.rabbit import RabbitSettings From 4cfa96df0257b4c782dddf40d6e90fb47637bffa Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 5 Jun 2025 16:06:04 +0200 Subject: [PATCH 06/46] refactor more imports --- .../fastapi/long_running_tasks/server.py | 18 ------------------ .../test_long_running_tasks_context_manager.py | 18 ++++++++---------- .../api/routes/dynamic_scheduler.py | 13 +++++-------- .../modules/dynamic_sidecar/scheduler/_abc.py | 3 +-- .../scheduler/_core/_events_utils.py | 3 +-- .../scheduler/_core/_scheduler.py | 3 +-- .../modules/dynamic_sidecar/scheduler/_task.py | 3 +-- .../api/rest/containers_long_running_tasks.py | 11 ++++------- .../src/simcore_service_dynamic_sidecar/cli.py | 2 +- 9 files changed, 22 insertions(+), 52 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py index e1decb5200f..51240434a77 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py @@ -6,30 +6,12 @@ running task. The client will take care of recovering the result from it. """ -from models_library.api_schemas_long_running_tasks.tasks import TaskResult - -from ...long_running_tasks.errors import TaskAlreadyRunningError, TaskCancelledError -from ...long_running_tasks.task import ( - TaskId, - TaskProgress, - TasksManager, - TaskStatus, - start_task, -) from ._dependencies import get_tasks_manager from ._server import setup __all__: tuple[str, ...] = ( "get_tasks_manager", "setup", - "start_task", - "TaskAlreadyRunningError", - "TaskCancelledError", - "TaskId", - "TasksManager", - "TaskProgress", - "TaskResult", - "TaskStatus", ) # nopycln: file diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index eac737268b5..cf9890a0ed4 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -13,20 +13,18 @@ from servicelib.fastapi.long_running_tasks._context_manager import _ProgressManager from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result from servicelib.fastapi.long_running_tasks.client import setup as setup_client -from servicelib.fastapi.long_running_tasks.server import ( - TaskId, - TaskProgress, - TasksManager, - get_tasks_manager, -) +from servicelib.fastapi.long_running_tasks.server import get_tasks_manager from servicelib.fastapi.long_running_tasks.server import setup as setup_server -from servicelib.fastapi.long_running_tasks.server import ( - start_task, -) from servicelib.long_running_tasks.errors import ( TaskClientTimeoutError, ) -from servicelib.long_running_tasks.models import ProgressMessage, ProgressPercent +from servicelib.long_running_tasks.models import ( + ProgressMessage, + ProgressPercent, + TaskId, + TaskProgress, +) +from servicelib.long_running_tasks.task import TasksManager, start_task TASK_SLEEP_INTERVAL: Final[PositiveFloat] = 0.1 diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py index 6934ce5056c..4f01e803dd0 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py @@ -4,18 +4,15 @@ from fastapi import APIRouter, Depends, HTTPException, status from models_library.projects_nodes_io import NodeID from pydantic import BaseModel, PositiveInt -from servicelib.fastapi.long_running_tasks.server import ( - TaskAlreadyRunningError, - TaskId, - TaskProgress, - TasksManager, - get_tasks_manager, - start_task, -) +from servicelib.fastapi.long_running_tasks.server import get_tasks_manager +from servicelib.long_running_tasks.errors import TaskAlreadyRunningError from servicelib.long_running_tasks.models import ( ProgressMessage, ProgressPercent, + TaskId, + TaskProgress, ) +from servicelib.long_running_tasks.task import TasksManager, start_task from tenacity import retry from tenacity.before_sleep import before_sleep_log from tenacity.retry import retry_if_result diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py index 869ea3d39f5..c5e47cc3c6b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py @@ -15,8 +15,7 @@ from models_library.services_types import ServicePortKey from models_library.users import UserID from models_library.wallets import WalletID -from servicelib.fastapi.long_running_tasks.server import TaskProgress -from servicelib.long_running_tasks.models import ProgressCallback +from servicelib.long_running_tasks.models import ProgressCallback, TaskProgress class SchedulerInternalsInterface(ABC): diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index 7579d7c1b67..6aa648b31e0 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -20,9 +20,8 @@ from models_library.user_preferences import FrontendUserPreference from models_library.users import UserID from servicelib.fastapi.http_client_thin import BaseHttpClientError -from servicelib.fastapi.long_running_tasks.server import TaskProgress from servicelib.logging_utils import log_context -from servicelib.long_running_tasks.models import ProgressCallback +from servicelib.long_running_tasks.models import ProgressCallback, TaskProgress from servicelib.rabbitmq import RabbitMQClient from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from servicelib.rabbitmq._errors import RemoteMethodNotRegisteredError diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index d3ef9ffc52d..ea36680240d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -43,8 +43,7 @@ from pydantic import NonNegativeFloat from servicelib.async_utils import cancel_wait_task from servicelib.background_task import create_periodic_task -from servicelib.fastapi.long_running_tasks.server import TaskProgress -from servicelib.long_running_tasks.models import ProgressCallback +from servicelib.long_running_tasks.models import ProgressCallback, TaskProgress from servicelib.redis import RedisClientsManager, exclusive from settings_library.redis import RedisDatabase diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py index 86592ebcaf9..555ea16a958 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py @@ -17,8 +17,7 @@ from models_library.services_types import ServicePortKey from models_library.users import UserID from models_library.wallets import WalletID -from servicelib.fastapi.long_running_tasks.server import TaskProgress -from servicelib.long_running_tasks.models import ProgressCallback +from servicelib.long_running_tasks.models import ProgressCallback, TaskProgress from ....core.dynamic_services_settings.scheduler import ( DynamicServicesSchedulerSettings, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py index af857013a82..da487ce392d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py @@ -2,14 +2,11 @@ from typing import Annotated, cast from fastapi import APIRouter, Depends, FastAPI, Request, status -from servicelib.fastapi.long_running_tasks.server import ( - TaskAlreadyRunningError, - TaskId, - TasksManager, - get_tasks_manager, - start_task, -) +from servicelib.fastapi.long_running_tasks.server import get_tasks_manager from servicelib.fastapi.requests_decorators import cancel_on_disconnect +from servicelib.long_running_tasks.errors import TaskAlreadyRunningError +from servicelib.long_running_tasks.models import TaskId +from servicelib.long_running_tasks.task import TasksManager, start_task from ...core.settings import ApplicationSettings from ...models.schemas.application_health import ApplicationHealth diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/cli.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/cli.py index 4bbf9e6016e..def22092390 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/cli.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/cli.py @@ -6,7 +6,7 @@ import typer from common_library.json_serialization import json_dumps from fastapi import FastAPI -from servicelib.fastapi.long_running_tasks.server import TaskProgress +from servicelib.long_running_tasks.models import TaskProgress from settings_library.utils_cli import create_settings_command from ._meta import PROJECT_NAME From 421f8cfbcf2a742f709911208ac333547c40a597 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 5 Jun 2025 16:10:31 +0200 Subject: [PATCH 07/46] refactor --- .../test_long_running_tasks.py | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index aee52da6fc8..7a5b919c34e 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -19,9 +19,18 @@ from fastapi import APIRouter, Depends, FastAPI, status from httpx import AsyncClient from pydantic import TypeAdapter -from servicelib.fastapi import long_running_tasks -from servicelib.long_running_tasks.models import TaskGet, TaskId, TaskProgress -from servicelib.long_running_tasks.task import TaskContext +from servicelib.fastapi.long_running_tasks.client import setup as setup_client +from servicelib.fastapi.long_running_tasks.server import ( + get_tasks_manager, +) +from servicelib.fastapi.long_running_tasks.server import setup as setup_server +from servicelib.long_running_tasks.models import ( + TaskGet, + TaskId, + TaskProgress, + TaskStatus, +) +from servicelib.long_running_tasks.task import TaskContext, TasksManager, start_task from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay @@ -43,7 +52,8 @@ async def _string_list_task( await asyncio.sleep(sleep_time) task_progress.update(message="generated item", percent=index / num_strings) if fail: - raise RuntimeError("We were asked to fail!!") + msg = "We were asked to fail!!" + raise RuntimeError(msg) return generated_strings @@ -59,11 +69,9 @@ async def create_string_list_task( num_strings: int, sleep_time: float, fail: bool = False, - task_manager: long_running_tasks.server.TasksManager = Depends( - long_running_tasks.server.get_tasks_manager - ), - ) -> long_running_tasks.server.TaskId: - task_id = long_running_tasks.server.start_task( + task_manager: TasksManager = Depends(get_tasks_manager), + ) -> TaskId: + task_id = start_task( task_manager, _string_list_task, num_strings=num_strings, @@ -80,8 +88,8 @@ async def app(server_routes: APIRouter) -> AsyncIterator[FastAPI]: # overrides fastapi/conftest.py:app app = FastAPI(title="test app") app.include_router(server_routes) - long_running_tasks.server.setup(app) - long_running_tasks.client.setup(app) + setup_server(app) + setup_client(app) async with LifespanManager(app): yield app @@ -94,10 +102,7 @@ async def _caller(app: FastAPI, client: AsyncClient, **query_kwargs) -> TaskId: ) resp = await client.post(f"{url}") assert resp.status_code == status.HTTP_202_ACCEPTED - task_id = TypeAdapter(long_running_tasks.server.TaskId).validate_python( - resp.json() - ) - return task_id + return TypeAdapter(TaskId).validate_python(resp.json()) return _caller @@ -124,9 +129,7 @@ async def _waiter( with attempt: result = await client.get(f"{status_url}") assert result.status_code == status.HTTP_200_OK - task_status = long_running_tasks.server.TaskStatus.model_validate( - result.json() - ) + task_status = TaskStatus.model_validate(result.json()) assert task_status assert task_status.done @@ -151,9 +154,7 @@ async def test_workflow( with attempt: result = await client.get(f"{status_url}") assert result.status_code == status.HTTP_200_OK - task_status = long_running_tasks.server.TaskStatus.model_validate( - result.json() - ) + task_status = TaskStatus.model_validate(result.json()) assert task_status progress_updates.append( (task_status.task_progress.message, task_status.task_progress.percent) From e73764173c7e8232b721b8aa7e18201abe587ea1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 6 Jun 2025 07:00:14 +0200 Subject: [PATCH 08/46] fixed import --- .../tests/integration/test_modules_long_running_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py index b7d45d90654..221f7f0a10f 100644 --- a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py @@ -29,7 +29,7 @@ from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from pytest_simcore.helpers.postgres_tools import PostgresTestConfig from pytest_simcore.helpers.storage import replace_storage_endpoint -from servicelib.fastapi.long_running_tasks.server import TaskProgress +from servicelib.long_running_tasks.models import TaskProgress from servicelib.utils import logged_gather from settings_library.s3 import S3Settings from simcore_postgres_database.models.projects import projects From ada50699bbdb554fd6a670acfdba2efd4eccd3bb Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 6 Jun 2025 07:03:02 +0200 Subject: [PATCH 09/46] fixed broken imports --- .../src/simcore_service_api_server/services_http/webserver.py | 2 +- services/web/server/tests/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/services_http/webserver.py b/services/api-server/src/simcore_service_api_server/services_http/webserver.py index 7751ffb382f..6ce01daccd3 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services_http/webserver.py @@ -41,11 +41,11 @@ from models_library.rest_pagination import Page, PageLimitInt, PageOffsetInt from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import PositiveInt -from servicelib.aiohttp.long_running_tasks.server import TaskStatus from servicelib.common_headers import ( X_SIMCORE_PARENT_NODE_ID, X_SIMCORE_PARENT_PROJECT_UUID, ) +from servicelib.long_running_tasks.models import TaskStatus from settings_library.tracing import TracingSettings from tenacity import TryAgain from tenacity.asyncio import AsyncRetrying diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index dc15b475984..fc85e684513 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -29,11 +29,11 @@ from pytest_simcore.helpers.webserver_login import LoggedUser, NewUser, UserInfoDict from pytest_simcore.simcore_webserver_projects_rest_api import NEW_PROJECT from servicelib.aiohttp import status -from servicelib.aiohttp.long_running_tasks.server import TaskStatus from servicelib.common_headers import ( X_SIMCORE_PARENT_NODE_ID, X_SIMCORE_PARENT_PROJECT_UUID, ) +from servicelib.long_running_tasks.models import TaskStatus from simcore_service_webserver.application_settings_utils import ( AppConfigDict, convert_to_environ_vars, From 7070b660da054d9839b6daa8be0986c5b8cb82a3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 6 Jun 2025 07:06:15 +0200 Subject: [PATCH 10/46] fixed imports --- .../cli/_close_and_save_service.py | 7 +++++-- .../modules/dynamic_sidecar/api_client/_public.py | 7 ++----- .../scheduler/_core/_events_user_services.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py b/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py index bffee5de248..d7f25a6a79d 100644 --- a/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py +++ b/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py @@ -16,11 +16,14 @@ from servicelib.fastapi.http_client_thin import UnexpectedStatusError from servicelib.fastapi.long_running_tasks.client import ( Client, - TaskId, periodic_task_result, setup, ) -from servicelib.long_running_tasks.models import ProgressMessage, ProgressPercent +from servicelib.long_running_tasks.models import ( + ProgressMessage, + ProgressPercent, + TaskId, +) from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_attempt diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py index 9b96137af3b..1dfb5b522d3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py @@ -22,16 +22,13 @@ BaseHttpClientError, UnexpectedStatusError, ) -from servicelib.fastapi.long_running_tasks.client import ( - Client, - TaskId, - periodic_task_result, -) +from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result from servicelib.logging_utils import log_context, log_decorator from servicelib.long_running_tasks.models import ( ProgressCallback, ProgressMessage, ProgressPercent, + TaskId, ) from servicelib.utils import logged_gather diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py index d7dd034134b..a0f543a83a3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py @@ -8,7 +8,7 @@ from models_library.services import ServiceVersion from models_library.services_creation import CreateServiceMetricsAdditionalParams from pydantic import TypeAdapter -from servicelib.fastapi.long_running_tasks.client import TaskId +from servicelib.long_running_tasks.models import TaskId from tenacity import RetryError from tenacity.asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log From 61daf54797f3b05ebeddb9576d0013bb1b36fbc3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 6 Jun 2025 07:08:47 +0200 Subject: [PATCH 11/46] fixeed imports --- .../projects/_controller/nodes_rest.py | 6 ++---- .../simcore_service_webserver/projects/_crud_api_create.py | 2 +- .../tests/unit/with_dbs/02/test_projects_cancellations.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py index c7f98d6e375..54125d4f9a8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py @@ -32,10 +32,7 @@ from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import BaseModel, Field from servicelib.aiohttp import status -from servicelib.aiohttp.long_running_tasks.server import ( - TaskProgress, - start_long_running_task, -) +from servicelib.aiohttp.long_running_tasks.server import start_long_running_task from servicelib.aiohttp.requests_validation import ( parse_request_body_as, parse_request_path_parameters_as, @@ -45,6 +42,7 @@ UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, X_SIMCORE_USER_AGENT, ) +from servicelib.long_running_tasks.models import TaskProgress from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from servicelib.rabbitmq import RPCServerError from servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.errors import ( diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 102707b2821..7ae8cb8654e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -17,7 +17,7 @@ from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.workspaces import UserWorkspaceWithAccessRights from pydantic import TypeAdapter -from servicelib.aiohttp.long_running_tasks.server import TaskProgress +from servicelib.long_running_tasks.models import TaskProgress from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from servicelib.redis import with_project_locked from servicelib.rest_constants import RESPONSE_MODEL_POLICY diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py index 07cb83015d0..61353b6f624 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py @@ -23,7 +23,7 @@ MockedStorageSubsystem, standard_role_response, ) -from servicelib.aiohttp.long_running_tasks.server import TaskGet +from servicelib.long_running_tasks.models import TaskGet from servicelib.rabbitmq.rpc_interfaces.async_jobs.async_jobs import ( AsyncJobComposedResult, ) From 75aa300385b2d0cded16ec4aac0aa99447875c1b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 6 Jun 2025 10:31:21 +0200 Subject: [PATCH 12/46] added missing specs --- api/specs/web-server/_long_running_tasks.py | 3 +- .../web-server/_long_running_tasks_legacy.py | 3 +- .../api/v0/openapi.yaml | 36 +++++++++++++++++-- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/api/specs/web-server/_long_running_tasks.py b/api/specs/web-server/_long_running_tasks.py index fc99db45b89..1d358cabf28 100644 --- a/api/specs/web-server/_long_running_tasks.py +++ b/api/specs/web-server/_long_running_tasks.py @@ -10,7 +10,7 @@ from models_library.generics import Envelope from models_library.rest_error import EnvelopedError from servicelib.aiohttp.long_running_tasks._routes import _PathParam -from servicelib.long_running_tasks.models import TaskGet, TaskStatus +from servicelib.long_running_tasks.models import TaskGet, TaskResult, TaskStatus from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.tasks._exception_handlers import ( _TO_HTTP_ERROR_MAP as export_data_http_error_map, @@ -65,6 +65,7 @@ def cancel_async_job( @router.get( "/tasks/{task_id}/result", + response_model=Envelope[TaskResult], name="get_task_result", description="Retrieves the result of a task", responses=_export_data_responses, diff --git a/api/specs/web-server/_long_running_tasks_legacy.py b/api/specs/web-server/_long_running_tasks_legacy.py index 89c85b0ff93..c6588b81d75 100644 --- a/api/specs/web-server/_long_running_tasks_legacy.py +++ b/api/specs/web-server/_long_running_tasks_legacy.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, status from models_library.generics import Envelope from servicelib.aiohttp.long_running_tasks._routes import _PathParam -from servicelib.long_running_tasks.models import TaskGet, TaskStatus +from servicelib.long_running_tasks.models import TaskGet, TaskResult, TaskStatus from simcore_service_webserver._meta import API_VTAG router = APIRouter( @@ -54,6 +54,7 @@ def cancel_and_delete_task( @router.get( "/{task_id}/result", name="get_task_result", + response_model=Envelope[TaskResult], description="Retrieves the result of a task", ) def get_task_result( diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 034cb621eba..72680936ccf 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3383,7 +3383,8 @@ paths: description: Successful Response content: application/json: - schema: {} + schema: + $ref: '#/components/schemas/Envelope_TaskResult_' '404': content: application/json: @@ -3478,7 +3479,8 @@ paths: description: Successful Response content: application/json: - schema: {} + schema: + $ref: '#/components/schemas/Envelope_TaskResult_' /v0/catalog/licensed-items: get: tags: @@ -10580,6 +10582,19 @@ components: title: Error type: object title: Envelope[TaskGet] + Envelope_TaskResult_: + properties: + data: + anyOf: + - $ref: '#/components/schemas/TaskResult' + - type: 'null' + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[TaskResult] Envelope_TaskStatus_: properties: data: @@ -16780,6 +16795,23 @@ components: to be defined as a float bound between 0.0 and 1.0' + TaskResult: + properties: + result: + anyOf: + - {} + - type: 'null' + title: Result + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + required: + - result + - error + title: TaskResult TaskStatus: properties: task_progress: From d94371070d231f4e000cdc90583698ff0439158e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 6 Jun 2025 10:43:36 +0200 Subject: [PATCH 13/46] removed unused code --- .../fastapi/long_running_tasks/client.py | 40 +------------------ 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py index e0eafea2257..b04f6092140 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py @@ -20,9 +20,8 @@ ) from yarl import URL -from ...long_running_tasks._constants import DEFAULT_POLL_INTERVAL_S, HOUR +from ...long_running_tasks._constants import DEFAULT_POLL_INTERVAL_S from ...long_running_tasks.models import ( - LRTask, RequestBody, TaskGet, TaskProgress, @@ -106,43 +105,6 @@ async def _abort_task(session: httpx.AsyncClient, abort_url: URL) -> None: response.raise_for_status() -async def long_running_task_request( - session: httpx.AsyncClient, - url: URL, - json: RequestBody | None = None, - client_timeout: int = 1 * HOUR, -) -> AsyncGenerator[LRTask, None]: - """Will use the passed `httpx.AsyncClient` to call an oSparc long - running task `url` passing `json` as request body. - NOTE: this follows the usual aiohttp client syntax, and will raise the same errors - - Raises: - [https://docs.aiohttp.org/en/stable/client_reference.html#hierarchy-of-exceptions] - """ - task = None - try: - task = await _start(session, url, json) - last_progress = None - async for task_progress in _wait_for_completion( - session, - task.task_id, - URL(task.status_href), - client_timeout, - ): - last_progress = task_progress - yield LRTask(progress=task_progress) - assert last_progress # nosec - yield LRTask( - progress=last_progress, - _result=_task_result(session, URL(task.result_href)), - ) - - except (TimeoutError, asyncio.CancelledError): - if task: - await _abort_task(session, URL(task.abort_href)) - raise - - __all__: tuple[str, ...] = ( "Client", "periodic_task_result", From 6a6677c49122ad34ad96183d9ba8691253e2e083 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 6 Jun 2025 10:46:48 +0200 Subject: [PATCH 14/46] remove unrequired --- .../src/servicelib/aiohttp/long_running_tasks/_routes.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 2603b229f68..7fb4520f117 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -90,11 +90,3 @@ async def cancel_and_delete_task(request: web.Request) -> web.Response: task_context = get_task_context(request) await tasks_manager.remove_task(path_params.task_id, with_task_context=task_context) return web.json_response(status=status.HTTP_204_NO_CONTENT) - - -__all__: tuple[str, ...] = ( - "get_tasks_manager", - "TaskId", - "TaskGet", - "TaskStatus", -) From 47e566778b2fc510c060241375fa922cde49dba2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 6 Jun 2025 11:03:24 +0200 Subject: [PATCH 15/46] removed unused --- .../aiohttp/long_running_tasks/_dependencies.py | 6 ++---- .../src/servicelib/aiohttp/long_running_tasks/_server.py | 8 ++++++-- .../src/servicelib/aiohttp/long_running_tasks/server.py | 2 -- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py index 4a8bc455cd8..b6b9a89d26a 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py @@ -8,6 +8,8 @@ RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, ) +# NOTE: figure out how to remove these and expose them differently if possible + def get_tasks_manager(app: web.Application) -> TasksManager: output: TasksManager = app[APP_LONG_RUNNING_TASKS_MANAGER_KEY] @@ -17,7 +19,3 @@ def get_tasks_manager(app: web.Application) -> TasksManager: def get_task_context(request: web.Request) -> dict[str, Any]: output: dict[str, Any] = request[RQT_LONG_RUNNING_TASKS_CONTEXT_KEY] return output - - -def create_task_name_from_request(request: web.Request) -> str: - return f"{request.method} {request.rel_url}" diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 40777376815..c7654592526 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -23,7 +23,7 @@ MINUTE, RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, ) -from ._dependencies import create_task_name_from_request, get_tasks_manager +from ._dependencies import get_tasks_manager from ._error_handlers import base_long_running_error_handler _logger = logging.getLogger(__name__) @@ -42,6 +42,10 @@ async def _wrap(request: web.Request): return _wrap +def _create_task_name_from_request(request: web.Request) -> str: + return f"{request.method} {request.rel_url}" + + async def start_long_running_task( # NOTE: positional argument are suffixed with "_" to avoid name conflicts with "task_kwargs" keys request_: web.Request, @@ -52,7 +56,7 @@ async def start_long_running_task( **task_kwargs: Any, ) -> web.Response: task_manager = get_tasks_manager(request_.app) - task_name = create_task_name_from_request(request_) + task_name = _create_task_name_from_request(request_) task_id = None try: task_id = start_task( diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py index 218fb4640f7..7ec90cb069e 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py @@ -7,14 +7,12 @@ """ from ._dependencies import ( - create_task_name_from_request, get_task_context, get_tasks_manager, ) from ._server import setup, start_long_running_task __all__: tuple[str, ...] = ( - "create_task_name_from_request", "get_task_context", "get_tasks_manager", "setup", From 87dec052e314d162e2b2f12c08a9e3b6aa8b3d9b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 10 Jun 2025 13:03:32 +0200 Subject: [PATCH 16/46] reverterd interface change --- api/specs/web-server/_long_running_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/specs/web-server/_long_running_tasks.py b/api/specs/web-server/_long_running_tasks.py index 1d358cabf28..9f3a1c4e6e5 100644 --- a/api/specs/web-server/_long_running_tasks.py +++ b/api/specs/web-server/_long_running_tasks.py @@ -10,7 +10,7 @@ from models_library.generics import Envelope from models_library.rest_error import EnvelopedError from servicelib.aiohttp.long_running_tasks._routes import _PathParam -from servicelib.long_running_tasks.models import TaskGet, TaskResult, TaskStatus +from servicelib.long_running_tasks.models import TaskGet, TaskResult from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.tasks._exception_handlers import ( _TO_HTTP_ERROR_MAP as export_data_http_error_map, @@ -41,7 +41,7 @@ def get_async_jobs(): ... @router.get( "/tasks/{task_id}", - response_model=Envelope[TaskStatus], + response_model=Any, name="get_task_status", description="Retrieves the status of a task", responses=_export_data_responses, From ebe6a43535dc85243001977014dda5348bb36d87 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 10 Jun 2025 13:04:20 +0200 Subject: [PATCH 17/46] reverted --- api/specs/web-server/_long_running_tasks_legacy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/specs/web-server/_long_running_tasks_legacy.py b/api/specs/web-server/_long_running_tasks_legacy.py index c6588b81d75..d17b9cceeed 100644 --- a/api/specs/web-server/_long_running_tasks_legacy.py +++ b/api/specs/web-server/_long_running_tasks_legacy.py @@ -4,12 +4,12 @@ # pylint: disable=too-many-arguments -from typing import Annotated +from typing import Annotated, Any from fastapi import APIRouter, Depends, status from models_library.generics import Envelope from servicelib.aiohttp.long_running_tasks._routes import _PathParam -from servicelib.long_running_tasks.models import TaskGet, TaskResult, TaskStatus +from servicelib.long_running_tasks.models import TaskGet, TaskStatus from simcore_service_webserver._meta import API_VTAG router = APIRouter( @@ -54,7 +54,7 @@ def cancel_and_delete_task( @router.get( "/{task_id}/result", name="get_task_result", - response_model=Envelope[TaskResult], + response_model=Any, description="Retrieves the result of a task", ) def get_task_result( From f630a395e69696015e6c5e0ba00b71681f6d5672 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 10 Jun 2025 13:06:48 +0200 Subject: [PATCH 18/46] updated openapi specs --- .../server/src/simcore_service_webserver/api/v0/openapi.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 72680936ccf..9db2fb69f01 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3299,7 +3299,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_TaskStatus_' + title: Response Get Async Job Status '404': content: application/json: @@ -3480,7 +3480,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_TaskResult_' + title: Response Get Task Result /v0/catalog/licensed-items: get: tags: From 757be743c1e696a19742996841e2ecbb64b90d6b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 10 Jun 2025 14:18:38 +0200 Subject: [PATCH 19/46] using commond part to format responses --- .../api_schemas_long_running_tasks/tasks.py | 5 +- .../aiohttp/long_running_tasks/_routes.py | 66 +++++++------------ .../fastapi/long_running_tasks/_routes.py | 37 +++++------ .../long_running_tasks/expose/__init__.py | 0 .../expose/endpoint_responses.py | 44 +++++++++++++ .../servicelib/long_running_tasks/models.py | 10 ++- 6 files changed, 90 insertions(+), 72 deletions(-) create mode 100644 packages/service-library/src/servicelib/long_running_tasks/expose/__init__.py create mode 100644 packages/service-library/src/servicelib/long_running_tasks/expose/endpoint_responses.py diff --git a/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py b/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py index acd73831b22..496151b36ed 100644 --- a/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py +++ b/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py @@ -18,9 +18,12 @@ class TaskResult(BaseModel): error: Any | None -class TaskGet(BaseModel): +class TaskGetWithoutHref(BaseModel): task_id: TaskId task_name: str + + +class TaskGet(TaskGetWithoutHref): status_href: str result_href: str abort_href: str diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 7fb4520f117..e5ab406af97 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -1,18 +1,15 @@ -import logging from typing import Any from aiohttp import web -from common_library.json_serialization import json_dumps from pydantic import BaseModel from servicelib.aiohttp import status +from servicelib.aiohttp.rest_responses import create_data_response -from ...long_running_tasks.errors import TaskNotCompletedError, TaskNotFoundError +from ...long_running_tasks.expose import endpoint_responses from ...long_running_tasks.models import TaskGet, TaskId, TaskStatus -from ...long_running_tasks.task import TrackedTask from ..requests_validation import parse_request_path_parameters_as from ._dependencies import get_task_context, get_tasks_manager -_logger = logging.getLogger(__name__) routes = web.RouteTableDef() @@ -24,24 +21,17 @@ class _PathParam(BaseModel): async def list_tasks(request: web.Request) -> web.Response: tasks_manager = get_tasks_manager(request.app) task_context = get_task_context(request) - tracked_tasks: list[TrackedTask] = tasks_manager.list_tasks( - with_task_context=task_context - ) - - return web.json_response( - { - "data": [ - TaskGet( - task_id=t.task_id, - task_name=t.task_name, - status_href=f"{request.app.router['get_task_status'].url_for(task_id=t.task_id)}", - result_href=f"{request.app.router['get_task_result'].url_for(task_id=t.task_id)}", - abort_href=f"{request.app.router['cancel_and_delete_task'].url_for(task_id=t.task_id)}", - ) - for t in tracked_tasks - ] - }, - dumps=json_dumps, + return create_data_response( + [ + TaskGet( + task_id=t.task_id, + task_name=t.task_name, + status_href=f"{request.app.router['get_task_status'].url_for(task_id=t.task_id)}", + result_href=f"{request.app.router['get_task_result'].url_for(task_id=t.task_id)}", + abort_href=f"{request.app.router['cancel_and_delete_task'].url_for(task_id=t.task_id)}", + ) + for t in endpoint_responses.list_tasks(tasks_manager, task_context) + ] ) @@ -51,10 +41,10 @@ async def get_task_status(request: web.Request) -> web.Response: tasks_manager = get_tasks_manager(request.app) task_context = get_task_context(request) - task_status: TaskStatus = tasks_manager.get_task_status( - task_id=path_params.task_id, with_task_context=task_context + task_status: TaskStatus = endpoint_responses.get_task_status( + tasks_manager, task_context, path_params.task_id ) - return web.json_response({"data": task_status}, dumps=json_dumps) + return create_data_response(task_status) @routes.get("/{task_id}/result", name="get_task_result") @@ -64,23 +54,9 @@ async def get_task_result(request: web.Request) -> web.Response | Any: task_context = get_task_context(request) # NOTE: this might raise an exception that will be catched by the _error_handlers - try: - task_result = tasks_manager.get_task_result( - task_id=path_params.task_id, with_task_context=task_context - ) - # NOTE: this will fail if the task failed for some reason.... - await tasks_manager.remove_task( - path_params.task_id, with_task_context=task_context, reraise_errors=False - ) - return task_result - except (TaskNotFoundError, TaskNotCompletedError): - raise - except Exception: - # the task shall be removed in this case - await tasks_manager.remove_task( - path_params.task_id, with_task_context=task_context, reraise_errors=False - ) - raise + return await endpoint_responses.get_task_result( + tasks_manager, task_context, path_params.task_id + ) @routes.delete("/{task_id}", name="cancel_and_delete_task") @@ -88,5 +64,7 @@ async def cancel_and_delete_task(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(_PathParam, request) tasks_manager = get_tasks_manager(request.app) task_context = get_task_context(request) - await tasks_manager.remove_task(path_params.task_id, with_task_context=task_context) + await endpoint_responses.remove_task( + tasks_manager, task_context, path_params.task_id + ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index fa11e506012..ea59ee095cb 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Depends, Request, status -from ...long_running_tasks.errors import TaskNotCompletedError, TaskNotFoundError +from ...long_running_tasks.expose import endpoint_responses from ...long_running_tasks.models import TaskGet, TaskId, TaskResult, TaskStatus from ...long_running_tasks.task import TasksManager from ..requests_decorators import cancel_on_disconnect @@ -21,11 +21,13 @@ async def list_tasks( TaskGet( task_id=t.task_id, task_name=t.task_name, - status_href="", - result_href="", - abort_href="", + status_href=str(request.url_for("get_task_status", task_id=t.task_id)), + result_href=str(request.url_for("get_task_result", task_id=t.task_id)), + abort_href=str( + request.url_for("cancel_and_delete_task", task_id=t.task_id) + ), ) - for t in tasks_manager.list_tasks(with_task_context=None) + for t in endpoint_responses.list_tasks(tasks_manager, task_context=None) ] @@ -43,7 +45,9 @@ async def get_task_status( tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], ) -> TaskStatus: assert request # nosec - return tasks_manager.get_task_status(task_id=task_id, with_task_context=None) + return endpoint_responses.get_task_status( + tasks_manager, task_context=None, task_id=task_id + ) @router.get( @@ -62,20 +66,9 @@ async def get_task_result( tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], ) -> TaskResult | Any: assert request # nosec - try: - task_result = tasks_manager.get_task_result(task_id, with_task_context=None) - await tasks_manager.remove_task( - task_id, with_task_context=None, reraise_errors=False - ) - return task_result - except (TaskNotFoundError, TaskNotCompletedError): - raise - except Exception: - # the task shall be removed in this case - await tasks_manager.remove_task( - task_id, with_task_context=None, reraise_errors=False - ) - raise + return await endpoint_responses.get_task_result( + tasks_manager, task_context=None, task_id=task_id + ) @router.delete( @@ -94,4 +87,6 @@ async def cancel_and_delete_task( tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], ) -> None: assert request # nosec - await tasks_manager.remove_task(task_id, with_task_context=None) + await endpoint_responses.remove_task( + tasks_manager, task_context=None, task_id=task_id + ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/expose/__init__.py b/packages/service-library/src/servicelib/long_running_tasks/expose/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/service-library/src/servicelib/long_running_tasks/expose/endpoint_responses.py b/packages/service-library/src/servicelib/long_running_tasks/expose/endpoint_responses.py new file mode 100644 index 00000000000..5599b906154 --- /dev/null +++ b/packages/service-library/src/servicelib/long_running_tasks/expose/endpoint_responses.py @@ -0,0 +1,44 @@ +from typing import Any + +from ..models import TaskGetWithoutHref, TaskId, TaskStatus +from ..task import TaskContext, TasksManager, TrackedTask + + +def list_tasks( + tasks_manager: TasksManager, task_context: TaskContext | None +) -> list[TaskGetWithoutHref]: + tracked_tasks: list[TrackedTask] = tasks_manager.list_tasks( + with_task_context=task_context + ) + return [ + TaskGetWithoutHref(task_id=t.task_id, task_name=t.task_name) + for t in tracked_tasks + ] + + +def get_task_status( + tasks_manager: TasksManager, task_context: TaskContext | None, task_id: TaskId +) -> TaskStatus: + return tasks_manager.get_task_status( + task_id=task_id, with_task_context=task_context + ) + + +async def get_task_result( + tasks_manager: TasksManager, task_context: TaskContext | None, task_id: TaskId +) -> Any: + try: + return tasks_manager.get_task_result( + task_id=task_id, with_task_context=task_context + ) + finally: + # the task is always removed even if an error occurs + await tasks_manager.remove_task( + task_id, with_task_context=task_context, reraise_errors=False + ) + + +async def remove_task( + tasks_manager: TasksManager, task_context: TaskContext | None, task_id: TaskId +) -> None: + await tasks_manager.remove_task(task_id, with_task_context=task_context) diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index 89fb8b1b399..00c86d025e0 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -13,6 +13,7 @@ ) from models_library.api_schemas_long_running_tasks.tasks import ( TaskGet, + TaskGetWithoutHref, TaskResult, TaskStatus, ) @@ -74,13 +75,8 @@ async def result(self) -> Any: return await self._result -# explicit export of models for api-schemas - -assert TaskResult # nosec -assert TaskGet # nosec -assert TaskStatus # nosec - __all__: tuple[str, ...] = ( + "TaskGetWithoutHref", "ProgressMessage", "ProgressPercent", "TaskGet", @@ -89,3 +85,5 @@ async def result(self) -> Any: "TaskResult", "TaskStatus", ) + +# nopycln: file From 8497191c04055ea68c49c762ef15e6358e1c6a6f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 10 Jun 2025 14:32:18 +0200 Subject: [PATCH 20/46] minor refacto --- .../servicelib/fastapi/long_running_tasks/_error_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_error_handlers.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_error_handlers.py index b39eff575bc..0214e009217 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_error_handlers.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_error_handlers.py @@ -18,10 +18,10 @@ async def base_long_running_error_handler( _: Request, exception: BaseLongRunningError ) -> JSONResponse: _logger.debug("%s", exception, stack_info=True) - error_fields = dict(code=exception.code, message=f"{exception}") + error_fields = {"code": exception.code, "message": f"{exception}"} status_code = ( status.HTTP_404_NOT_FOUND - if isinstance(exception, (TaskNotFoundError, TaskNotCompletedError)) + if isinstance(exception, TaskNotFoundError | TaskNotCompletedError) else status.HTTP_400_BAD_REQUEST ) return JSONResponse(content=jsonable_encoder(error_fields), status_code=status_code) From 866656de25f499de3248cc0d62adf38d423b08a7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 10 Jun 2025 14:32:30 +0200 Subject: [PATCH 21/46] removed unused --- .../fastapi/long_running_tasks/_client.py | 11 --- .../fastapi/long_running_tasks/client.py | 95 ------------------- 2 files changed, 106 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py index c2b93594ce1..a344e6b09a1 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py @@ -1,7 +1,6 @@ import asyncio import functools import logging -import warnings from collections.abc import Awaitable, Callable from typing import Any, Final @@ -181,16 +180,6 @@ async def cancel_and_delete_task( timeout=timeout, ) - if result.status_code == status.HTTP_200_OK: - warnings.warn( - "returning a 200 when cancelling a task has been deprecated with PR#3236" - "and will be removed after 11.2022" - "please do close your studies at least once before that date, so that the dy-sidecar" - "get replaced", - category=DeprecationWarning, - ) - return - if result.status_code not in ( status.HTTP_204_NO_CONTENT, status.HTTP_404_NOT_FOUND, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py index b04f6092140..ccc8b815fea 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py @@ -2,109 +2,14 @@ Provides a convenient way to return the result given a TaskId. """ -import asyncio import logging -from collections.abc import AsyncGenerator -from typing import Any -import httpx -from fastapi import status -from tenacity import ( - AsyncRetrying, - TryAgain, - before_sleep_log, - retry, - retry_if_exception_type, - stop_after_delay, - wait_random_exponential, -) -from yarl import URL - -from ...long_running_tasks._constants import DEFAULT_POLL_INTERVAL_S -from ...long_running_tasks.models import ( - RequestBody, - TaskGet, - TaskProgress, - TaskStatus, -) -from ...long_running_tasks.task import TaskId -from ...rest_responses import unwrap_envelope_if_required from ._client import Client, setup from ._context_manager import periodic_task_result _logger = logging.getLogger(__name__) -_DEFAULT_FASTAPI_RETRY_POLICY: dict[str, Any] = { - "retry": retry_if_exception_type(httpx.RequestError), - "wait": wait_random_exponential(max=20), - "stop": stop_after_delay(60), - "reraise": True, - "before_sleep": before_sleep_log(_logger, logging.INFO), -} - - -@retry(**_DEFAULT_FASTAPI_RETRY_POLICY) -async def _start( - session: httpx.AsyncClient, url: URL, json: RequestBody | None -) -> TaskGet: - response = await session.post(f"{url}", json=json) - response.raise_for_status() - data = unwrap_envelope_if_required(response.json()) - return TaskGet.model_validate(data) - - -@retry(**_DEFAULT_FASTAPI_RETRY_POLICY) -async def _wait_for_completion( - session: httpx.AsyncClient, - task_id: TaskId, - status_url: URL, - client_timeout: int, -) -> AsyncGenerator[TaskProgress, None]: - try: - async for attempt in AsyncRetrying( - stop=stop_after_delay(client_timeout), - reraise=True, - retry=retry_if_exception_type(TryAgain), - before_sleep=before_sleep_log(_logger, logging.DEBUG), - ): - with attempt: - response = await session.get(f"{status_url}") - response.raise_for_status() - data = unwrap_envelope_if_required(response.json()) - task_status = TaskStatus.model_validate(data) - - yield task_status.task_progress - if not task_status.done: - await asyncio.sleep( - float( - response.headers.get("retry-after", DEFAULT_POLL_INTERVAL_S) - ) - ) - msg = f"{task_id=}, {task_status.started=} has status: '{task_status.task_progress.message}' {task_status.task_progress.percent}%" - raise TryAgain(msg) # noqa: TRY301 - - except TryAgain as exc: - # this is a timeout - msg = f"Long running task {task_id}, calling to {status_url} timed-out after {client_timeout} seconds" - raise TimeoutError(msg) from exc - - -@retry(**_DEFAULT_FASTAPI_RETRY_POLICY) -async def _task_result(session: httpx.AsyncClient, result_url: URL) -> Any: - response = await session.get(f"{result_url}") - response.raise_for_status() - if response.status_code != status.HTTP_204_NO_CONTENT: - return unwrap_envelope_if_required(response.json()) - return None - - -@retry(**_DEFAULT_FASTAPI_RETRY_POLICY) -async def _abort_task(session: httpx.AsyncClient, abort_url: URL) -> None: - response = await session.delete(f"{abort_url}") - response.raise_for_status() - - __all__: tuple[str, ...] = ( "Client", "periodic_task_result", From 2ee9203b14e55c0f0a319270186845d46af9c9be Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 10 Jun 2025 15:33:04 +0200 Subject: [PATCH 22/46] refactor to use common constants --- .../aiohttp/long_running_tasks/_server.py | 12 +++++--- .../aiohttp/long_running_tasks/client.py | 2 +- .../fastapi/long_running_tasks/_server.py | 13 ++++---- .../long_running_tasks/_constants.py | 5 ---- .../long_running_tasks/constants.py | 9 ++++++ .../src/servicelib/long_running_tasks/task.py | 30 +++++++++++-------- 6 files changed, 43 insertions(+), 28 deletions(-) delete mode 100644 packages/service-library/src/servicelib/long_running_tasks/_constants.py create mode 100644 packages/service-library/src/servicelib/long_running_tasks/constants.py diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index c7654592526..3e025a08354 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -1,4 +1,5 @@ import asyncio +import datetime import logging from collections.abc import AsyncGenerator, Callable from functools import wraps @@ -6,9 +7,13 @@ from aiohttp import web from common_library.json_serialization import json_dumps -from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter +from pydantic import AnyHttpUrl, TypeAdapter from ...aiohttp import status +from ...long_running_tasks.constants import ( + DEFAULT_STALE_TASK_CHECK_INTERVAL, + DEFAULT_STALE_TASK_DETECT_TIMEOUT, +) from ...long_running_tasks.models import TaskGet from ...long_running_tasks.task import ( TaskContext, @@ -20,7 +25,6 @@ from . import _routes from ._constants import ( APP_LONG_RUNNING_TASKS_MANAGER_KEY, - MINUTE, RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, ) from ._dependencies import get_tasks_manager @@ -125,8 +129,8 @@ def setup( router_prefix: str, handler_check_decorator: Callable = no_ops_decorator, task_request_context_decorator: Callable = no_task_context_decorator, - stale_task_check_interval_s: PositiveFloat = 1 * MINUTE, - stale_task_detect_timeout_s: PositiveFloat = 5 * MINUTE, + stale_task_check_interval_s: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, + stale_task_detect_timeout_s: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -> None: """ - `router_prefix` APIs are mounted on `/...`, this diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py index 81c48ed017b..34a2de06227 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py @@ -12,7 +12,7 @@ from tenacity.wait import wait_random_exponential from yarl import URL -from ...long_running_tasks._constants import DEFAULT_POLL_INTERVAL_S, HOUR +from ...long_running_tasks.constants import DEFAULT_POLL_INTERVAL_S, HOUR from ...long_running_tasks.models import ( LRTask, RequestBody, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index 26c26d10bc3..9ffd135c9d8 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -1,22 +1,23 @@ -from typing import Final +import datetime from fastapi import APIRouter, FastAPI -from pydantic import PositiveFloat +from ...long_running_tasks.constants import ( + DEFAULT_STALE_TASK_CHECK_INTERVAL, + DEFAULT_STALE_TASK_DETECT_TIMEOUT, +) from ...long_running_tasks.errors import BaseLongRunningError from ...long_running_tasks.task import TasksManager from ._error_handlers import base_long_running_error_handler from ._routes import router -_MINUTE: Final[PositiveFloat] = 60 - def setup( app: FastAPI, *, router_prefix: str = "", - stale_task_check_interval_s: PositiveFloat = 1 * _MINUTE, - stale_task_detect_timeout_s: PositiveFloat = 5 * _MINUTE, + stale_task_check_interval_s: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, + stale_task_detect_timeout_s: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -> None: """ - `router_prefix` APIs are mounted on `/task/...`, this diff --git a/packages/service-library/src/servicelib/long_running_tasks/_constants.py b/packages/service-library/src/servicelib/long_running_tasks/_constants.py deleted file mode 100644 index 5cc87208a36..00000000000 --- a/packages/service-library/src/servicelib/long_running_tasks/_constants.py +++ /dev/null @@ -1,5 +0,0 @@ -from typing import Final - -MINUTE: Final[int] = 60 # in secs -HOUR: Final[int] = 60 * MINUTE # in secs -DEFAULT_POLL_INTERVAL_S: Final[float] = 1 diff --git a/packages/service-library/src/servicelib/long_running_tasks/constants.py b/packages/service-library/src/servicelib/long_running_tasks/constants.py new file mode 100644 index 00000000000..45567e3e80b --- /dev/null +++ b/packages/service-library/src/servicelib/long_running_tasks/constants.py @@ -0,0 +1,9 @@ +from datetime import timedelta +from typing import Final + +MINUTE: Final[int] = 60 # in secs +HOUR: Final[int] = 60 * MINUTE # in secs +DEFAULT_POLL_INTERVAL_S: Final[float] = 1 + +DEFAULT_STALE_TASK_CHECK_INTERVAL: Final[timedelta] = timedelta(minutes=1) +DEFAULT_STALE_TASK_DETECT_TIMEOUT: Final[timedelta] = timedelta(minutes=5) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 5e02b1adbec..d2f560df8e5 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -1,12 +1,12 @@ import asyncio +import datetime import inspect import logging import traceback import urllib.parse from collections import deque from contextlib import suppress -from datetime import datetime -from typing import Any, Protocol +from typing import Any, Final, Protocol from uuid import uuid4 from models_library.api_schemas_long_running_tasks.base import ( @@ -26,6 +26,10 @@ logger = logging.getLogger(__name__) +_CANCEL_TASK_TIMEOUT: Final[PositiveFloat] = datetime.timedelta( + seconds=1 +).total_seconds() + async def _await_task(task: asyncio.Task) -> None: await task @@ -35,7 +39,7 @@ def _mark_task_to_remove_if_required( task_id: TaskId, tasks_to_remove: list[TaskId], tracked_task: TrackedTask, - utc_now: datetime, + utc_now: datetime.datetime, stale_timeout_s: float, ) -> None: if tracked_task.fire_and_forget: @@ -64,16 +68,18 @@ class TasksManager: def __init__( self, - stale_task_check_interval_s: PositiveFloat, - stale_task_detect_timeout_s: PositiveFloat, + stale_task_check_interval_s: datetime.timedelta, + stale_task_detect_timeout_s: datetime.timedelta, ): # Task groups: Every taskname maps to multiple asyncio.Task within TrackedTask model self._tasks_groups: dict[TaskName, TrackedTaskGroupDict] = {} - self._cancel_task_timeout_s: PositiveFloat = 1.0 - - self.stale_task_check_interval_s = stale_task_check_interval_s - self.stale_task_detect_timeout_s = stale_task_detect_timeout_s + self.stale_task_check_interval_s: PositiveFloat = ( + stale_task_check_interval_s.total_seconds() + ) + self.stale_task_detect_timeout_s: PositiveFloat = ( + stale_task_detect_timeout_s.total_seconds() + ) self._stale_tasks_monitor_task: asyncio.Task = asyncio.create_task( self._stale_tasks_monitor_worker(), name=f"{__name__}.stale_task_monitor_worker", @@ -100,7 +106,7 @@ async def _stale_tasks_monitor_worker(self) -> None: # will not be the case. while await asyncio.sleep(self.stale_task_check_interval_s, result=True): - utc_now = datetime.utcnow() + utc_now = datetime.datetime.now(tz=datetime.UTC) tasks_to_remove: list[TaskId] = [] for tasks in self._tasks_groups.values(): @@ -207,7 +213,7 @@ def get_task_status( raises TaskNotFoundError if the task cannot be found """ tracked_task: TrackedTask = self._get_tracked_task(task_id, with_task_context) - tracked_task.last_status_check = datetime.utcnow() + tracked_task.last_status_check = datetime.datetime.now(tz=datetime.UTC) task = tracked_task.task done = task.done() @@ -260,7 +266,7 @@ async def _cancel_asyncio_task( try: try: await asyncio.wait_for( - _await_task(task), timeout=self._cancel_task_timeout_s + _await_task(task), timeout=_CANCEL_TASK_TIMEOUT ) except TimeoutError: logger.warning( From d886db03431bead65fdd8eab2786db200c12fed3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 10 Jun 2025 15:59:11 +0200 Subject: [PATCH 23/46] using common renaming --- .../aiohttp/long_running_tasks/_server.py | 8 +++--- .../fastapi/long_running_tasks/_server.py | 8 +++--- .../servicelib/long_running_tasks/models.py | 6 ++--- .../src/servicelib/long_running_tasks/task.py | 26 +++++++++---------- .../test_long_running_tasks_task.py | 6 ++--- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 3e025a08354..b570b5fdd50 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -129,8 +129,8 @@ def setup( router_prefix: str, handler_check_decorator: Callable = no_ops_decorator, task_request_context_decorator: Callable = no_task_context_decorator, - stale_task_check_interval_s: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, - stale_task_detect_timeout_s: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, + stale_task_check_interval: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, + stale_task_detect_timeout: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -> None: """ - `router_prefix` APIs are mounted on `/...`, this @@ -146,8 +146,8 @@ async def on_cleanup_ctx(app: web.Application) -> AsyncGenerator[None, None]: # add components to state app[APP_LONG_RUNNING_TASKS_MANAGER_KEY] = long_running_task_manager = ( TasksManager( - stale_task_check_interval_s=stale_task_check_interval_s, - stale_task_detect_timeout_s=stale_task_detect_timeout_s, + stale_task_check_interval=stale_task_check_interval, + stale_task_detect_timeout=stale_task_detect_timeout, ) ) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index 9ffd135c9d8..4682041929f 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -16,8 +16,8 @@ def setup( app: FastAPI, *, router_prefix: str = "", - stale_task_check_interval_s: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, - stale_task_detect_timeout_s: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, + stale_task_check_interval: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, + stale_task_detect_timeout: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -> None: """ - `router_prefix` APIs are mounted on `/task/...`, this @@ -37,8 +37,8 @@ async def on_startup() -> None: # add components to state app.state.long_running_task_manager = TasksManager( - stale_task_check_interval_s=stale_task_check_interval_s, - stale_task_detect_timeout_s=stale_task_detect_timeout_s, + stale_task_check_interval=stale_task_check_interval, + stale_task_detect_timeout=stale_task_detect_timeout, ) async def on_shutdown() -> None: diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index 00c86d025e0..deb3d0e5d6e 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -2,7 +2,7 @@ from asyncio import Task from collections.abc import Awaitable, Callable, Coroutine from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime from typing import Any, TypeAlias from models_library.api_schemas_long_running_tasks.base import ( @@ -42,7 +42,7 @@ class TrackedTask(BaseModel): description="if True then the task will not be auto-cancelled if no one enquires of its status", ) - started: datetime = Field(default_factory=datetime.utcnow) + started: datetime = Field(default_factory=lambda: datetime.now(UTC)) last_status_check: datetime | None = Field( default=None, description=( @@ -76,10 +76,10 @@ async def result(self) -> Any: __all__: tuple[str, ...] = ( - "TaskGetWithoutHref", "ProgressMessage", "ProgressPercent", "TaskGet", + "TaskGetWithoutHref", "TaskId", "TaskProgress", "TaskResult", diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index d2f560df8e5..0735bfbf246 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -9,10 +9,7 @@ from typing import Any, Final, Protocol from uuid import uuid4 -from models_library.api_schemas_long_running_tasks.base import ( - ProgressPercent, - TaskProgress, -) +from models_library.api_schemas_long_running_tasks.base import TaskProgress from pydantic import PositiveFloat from .errors import ( @@ -24,7 +21,7 @@ ) from .models import TaskId, TaskName, TaskStatus, TrackedTask -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) _CANCEL_TASK_TIMEOUT: Final[PositiveFloat] = datetime.timedelta( seconds=1 @@ -68,22 +65,23 @@ class TasksManager: def __init__( self, - stale_task_check_interval_s: datetime.timedelta, - stale_task_detect_timeout_s: datetime.timedelta, + stale_task_check_interval: datetime.timedelta, + stale_task_detect_timeout: datetime.timedelta, ): # Task groups: Every taskname maps to multiple asyncio.Task within TrackedTask model self._tasks_groups: dict[TaskName, TrackedTaskGroupDict] = {} self.stale_task_check_interval_s: PositiveFloat = ( - stale_task_check_interval_s.total_seconds() + stale_task_check_interval.total_seconds() ) self.stale_task_detect_timeout_s: PositiveFloat = ( - stale_task_detect_timeout_s.total_seconds() + stale_task_detect_timeout.total_seconds() ) self._stale_tasks_monitor_task: asyncio.Task = asyncio.create_task( self._stale_tasks_monitor_worker(), name=f"{__name__}.stale_task_monitor_worker", ) + # TODO: add setup and teardown and also use periodic def get_task_group(self, task_name: TaskName) -> TrackedTaskGroupDict: return self._tasks_groups[task_name] @@ -126,7 +124,7 @@ async def _stale_tasks_monitor_worker(self) -> None: # - finished with a result # - finished with errors # we just print the status from where one can infer the above - logger.warning( + _logger.warning( "Removing stale task '%s' with status '%s'", task_id, self.get_task_status( @@ -269,7 +267,7 @@ async def _cancel_asyncio_task( _await_task(task), timeout=_CANCEL_TASK_TIMEOUT ) except TimeoutError: - logger.warning( + _logger.warning( "Timed out while awaiting for cancellation of '%s'", reference ) except Exception: # pylint:disable=broad-except @@ -395,11 +393,11 @@ def start_task( # bind the task with progress 0 and 1 async def _progress_task(progress: TaskProgress, handler: TaskProtocol): - progress.update(message="starting", percent=ProgressPercent(0)) + progress.update(message="starting", percent=0) try: return await handler(progress, **task_kwargs) finally: - progress.update(message="finished", percent=ProgressPercent(1)) + progress.update(message="finished", percent=1) async_task = asyncio.create_task( _progress_task(task_progress, task), name=f"{task_name}" @@ -421,9 +419,9 @@ async def _progress_task(progress: TaskProgress, handler: TaskProtocol): "TaskAlreadyRunningError", "TaskCancelledError", "TaskId", - "TasksManager", "TaskProgress", "TaskProtocol", "TaskStatus", + "TasksManager", "TrackedTask", ) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 63be0eb1262..b14d4f7de5f 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -7,7 +7,7 @@ import asyncio import urllib.parse from collections.abc import AsyncIterator -from datetime import datetime +from datetime import datetime, timedelta from typing import Any, Final import pytest @@ -71,8 +71,8 @@ async def failing_background_task(task_progress: TaskProgress): @pytest.fixture async def tasks_manager() -> AsyncIterator[TasksManager]: tasks_manager = TasksManager( - stale_task_check_interval_s=TEST_CHECK_STALE_INTERVAL_S, - stale_task_detect_timeout_s=TEST_CHECK_STALE_INTERVAL_S, + stale_task_check_interval=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), + stale_task_detect_timeout=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), ) yield tasks_manager await tasks_manager.close() From ecd19870a3fb1624015a06bb5b9c19408058b91c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 08:25:44 +0200 Subject: [PATCH 24/46] fixed imports --- .../service-library/src/servicelib/long_running_tasks/task.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 252b1128430..5870ab61992 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -6,7 +6,6 @@ import urllib.parse from collections import deque from contextlib import suppress -from datetime import datetime from typing import Any, Final, Protocol, TypeAlias from uuid import uuid4 From ea17b89460fb1cdb2366a237b3588da76bc13913 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 08:28:26 +0200 Subject: [PATCH 25/46] removed unused --- .../src/servicelib/fastapi/long_running_tasks/client.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py index ccc8b815fea..ab2ecb1a73f 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/client.py @@ -2,13 +2,8 @@ Provides a convenient way to return the result given a TaskId. """ -import logging - from ._client import Client, setup -from ._context_manager import periodic_task_result - -_logger = logging.getLogger(__name__) - +from ._context_manager import periodic_task_result # attach to the same object! __all__: tuple[str, ...] = ( "Client", From 0578e5ca3aa06c66b86b7d0cca7a02034b754eef Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 08:42:56 +0200 Subject: [PATCH 26/46] refacto tasks --- .../aiohttp/long_running_tasks/_server.py | 8 +- .../src/servicelib/long_running_tasks/task.py | 107 +++++++++--------- .../test_long_running_tasks_task.py | 3 +- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index b570b5fdd50..da33ecd3562 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -143,6 +143,9 @@ def setup( """ async def on_cleanup_ctx(app: web.Application) -> AsyncGenerator[None, None]: + # add error handlers + app.middlewares.append(base_long_running_error_handler) + # add components to state app[APP_LONG_RUNNING_TASKS_MANAGER_KEY] = long_running_task_manager = ( TasksManager( @@ -151,13 +154,12 @@ async def on_cleanup_ctx(app: web.Application) -> AsyncGenerator[None, None]: ) ) - # add error handlers - app.middlewares.append(base_long_running_error_handler) + await long_running_task_manager.setup() yield # cleanup - await long_running_task_manager.close() + await long_running_task_manager.teardown() # add routing (done at setup-time) _wrap_and_add_routes( diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 5870ab61992..8e98413e72d 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -11,6 +11,9 @@ from models_library.api_schemas_long_running_tasks.base import TaskProgress from pydantic import PositiveFloat +from servicelib.async_utils import cancel_wait_task +from servicelib.background_task import create_periodic_task +from servicelib.logging_utils import log_catch from .errors import ( TaskAlreadyRunningError, @@ -71,17 +74,36 @@ def __init__( # Task groups: Every taskname maps to multiple asyncio.Task within TrackedTask model self._tasks_groups: dict[TaskName, TrackedTaskGroupDict] = {} - self.stale_task_check_interval_s: PositiveFloat = ( - stale_task_check_interval.total_seconds() - ) + self.stale_task_check_interval: datetime.timedelta = stale_task_check_interval self.stale_task_detect_timeout_s: PositiveFloat = ( stale_task_detect_timeout.total_seconds() ) - self._stale_tasks_monitor_task: asyncio.Task = asyncio.create_task( - self._stale_tasks_monitor_worker(), - name=f"{__name__}.stale_task_monitor_worker", + + self._stale_tasks_monitor_task: asyncio.Task | None = None + + async def setup(self) -> None: + self._stale_tasks_monitor_task = create_periodic_task( + task=self._stale_tasks_monitor_worker, + interval=self.stale_task_check_interval, + task_name=f"{__name__}.stale_task_monitor_worker", ) - # TODO: add setup and teardown and also use periodic + + async def teardown(self) -> None: + task_ids_to_remove: deque[TaskId] = deque() + + for tasks_dict in self._tasks_groups.values(): + for tracked_task in tasks_dict.values(): + task_ids_to_remove.append(tracked_task.task_id) + + for task_id in task_ids_to_remove: + # when closing we do not care about pending errors + await self.remove_task(task_id, None, reraise_errors=False) + + if self._stale_tasks_monitor_task: + with log_catch(_logger, reraise=False): + await cancel_wait_task( + self._stale_tasks_monitor_task, max_delay=_CANCEL_TASK_TIMEOUT + ) def get_task_group(self, task_name: TaskName) -> TrackedTaskGroupDict: return self._tasks_groups[task_name] @@ -103,38 +125,35 @@ async def _stale_tasks_monitor_worker(self) -> None: # Since we own the client, we assume (for now) this # will not be the case. - while await asyncio.sleep(self.stale_task_check_interval_s, result=True): - utc_now = datetime.datetime.now(tz=datetime.UTC) - - tasks_to_remove: list[TaskId] = [] - for tasks in self._tasks_groups.values(): - for task_id, tracked_task in tasks.items(): - _mark_task_to_remove_if_required( - task_id, - tasks_to_remove, - tracked_task, - utc_now, - self.stale_task_detect_timeout_s, - ) + utc_now = datetime.datetime.now(tz=datetime.UTC) - # finally remove tasks and warn - for task_id in tasks_to_remove: - # NOTE: task can be in the following cases: - # - still ongoing - # - finished with a result - # - finished with errors - # we just print the status from where one can infer the above - _logger.warning( - "Removing stale task '%s' with status '%s'", + tasks_to_remove: list[TaskId] = [] + for tasks in self._tasks_groups.values(): + for task_id, tracked_task in tasks.items(): + _mark_task_to_remove_if_required( task_id, - self.get_task_status( - task_id, with_task_context=None - ).model_dump_json(), - ) - await self.remove_task( - task_id, with_task_context=None, reraise_errors=False + tasks_to_remove, + tracked_task, + utc_now, + self.stale_task_detect_timeout_s, ) + # finally remove tasks and warn + for task_id in tasks_to_remove: + # NOTE: task can be in the following cases: + # - still ongoing + # - finished with a result + # - finished with errors + # we just print the status from where one can infer the above + _logger.warning( + "Removing stale task '%s' with status '%s'", + task_id, + self.get_task_status(task_id, with_task_context=None).model_dump_json(), + ) + await self.remove_task( + task_id, with_task_context=None, reraise_errors=False + ) + @staticmethod def create_task_id(task_name: TaskName) -> str: assert len(task_name) > 0 @@ -308,24 +327,6 @@ async def remove_task( finally: del self._tasks_groups[tracked_task.task_name][task_id] - async def close(self) -> None: - """ - cancels all pending tasks and removes them before closing - """ - task_ids_to_remove: deque[TaskId] = deque() - - for tasks_dict in self._tasks_groups.values(): - for tracked_task in tasks_dict.values(): - task_ids_to_remove.append(tracked_task.task_id) - - for task_id in task_ids_to_remove: - # when closing we do not care about pending errors - await self.remove_task(task_id, None, reraise_errors=False) - - await self._cancel_asyncio_task( - self._stale_tasks_monitor_task, "stale_monitor", reraise_errors=False - ) - class TaskProtocol(Protocol): async def __call__( diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index b14d4f7de5f..6e3ec1522e2 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -74,8 +74,9 @@ async def tasks_manager() -> AsyncIterator[TasksManager]: stale_task_check_interval=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), stale_task_detect_timeout=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), ) + await tasks_manager.setup() yield tasks_manager - await tasks_manager.close() + await tasks_manager.teardown() @pytest.mark.parametrize("check_task_presence_before", [True, False]) From a4660de5fe1d191bd7cb11a8a3dc118b1e9598f8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 08:45:08 +0200 Subject: [PATCH 27/46] minor --- .../service-library/src/servicelib/long_running_tasks/task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 8e98413e72d..95240baf6ae 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -74,7 +74,7 @@ def __init__( # Task groups: Every taskname maps to multiple asyncio.Task within TrackedTask model self._tasks_groups: dict[TaskName, TrackedTaskGroupDict] = {} - self.stale_task_check_interval: datetime.timedelta = stale_task_check_interval + self.stale_task_check_interval = stale_task_check_interval self.stale_task_detect_timeout_s: PositiveFloat = ( stale_task_detect_timeout.total_seconds() ) @@ -85,7 +85,7 @@ async def setup(self) -> None: self._stale_tasks_monitor_task = create_periodic_task( task=self._stale_tasks_monitor_worker, interval=self.stale_task_check_interval, - task_name=f"{__name__}.stale_task_monitor_worker", + task_name=f"{__name__}.{self._stale_tasks_monitor_worker.__name__}", ) async def teardown(self) -> None: From 70a6921732d60dd869ac2ab968abf3f81b14c66b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 08:48:18 +0200 Subject: [PATCH 28/46] revert changes --- api/specs/web-server/_long_running_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/specs/web-server/_long_running_tasks.py b/api/specs/web-server/_long_running_tasks.py index 9f3a1c4e6e5..1c6e033b867 100644 --- a/api/specs/web-server/_long_running_tasks.py +++ b/api/specs/web-server/_long_running_tasks.py @@ -10,7 +10,7 @@ from models_library.generics import Envelope from models_library.rest_error import EnvelopedError from servicelib.aiohttp.long_running_tasks._routes import _PathParam -from servicelib.long_running_tasks.models import TaskGet, TaskResult +from servicelib.long_running_tasks.models import TaskGet, TaskStatus from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.tasks._exception_handlers import ( _TO_HTTP_ERROR_MAP as export_data_http_error_map, @@ -41,7 +41,7 @@ def get_async_jobs(): ... @router.get( "/tasks/{task_id}", - response_model=Any, + response_model=Envelope[TaskStatus], name="get_task_status", description="Retrieves the status of a task", responses=_export_data_responses, @@ -65,7 +65,7 @@ def cancel_async_job( @router.get( "/tasks/{task_id}/result", - response_model=Envelope[TaskResult], + response_model=Any, name="get_task_result", description="Retrieves the result of a task", responses=_export_data_responses, From 8374de5059319dc33ca459468a485dd6b22b7142 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 09:07:39 +0200 Subject: [PATCH 29/46] fixed specs --- .../api/v0/openapi.yaml | 34 ++----------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 9db2fb69f01..c8ed429c797 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3299,7 +3299,7 @@ paths: content: application/json: schema: - title: Response Get Async Job Status + $ref: '#/components/schemas/Envelope_TaskStatus_' '404': content: application/json: @@ -3384,7 +3384,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_TaskResult_' + title: Response Get Async Job Result '404': content: application/json: @@ -10582,19 +10582,6 @@ components: title: Error type: object title: Envelope[TaskGet] - Envelope_TaskResult_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/TaskResult' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[TaskResult] Envelope_TaskStatus_: properties: data: @@ -16795,23 +16782,6 @@ components: to be defined as a float bound between 0.0 and 1.0' - TaskResult: - properties: - result: - anyOf: - - {} - - type: 'null' - title: Result - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - required: - - result - - error - title: TaskResult TaskStatus: properties: task_progress: From 221e051d3a2ac271e88938e3deecfdfbb3a8e5ef Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 09:30:33 +0200 Subject: [PATCH 30/46] fixed failing tests --- .../src/servicelib/fastapi/long_running_tasks/_server.py | 2 +- services/dynamic-sidecar/tests/unit/test_api_rest_containers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index 4682041929f..6afa935b013 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -44,7 +44,7 @@ async def on_startup() -> None: async def on_shutdown() -> None: if app.state.long_running_task_manager: task_manager: TasksManager = app.state.long_running_task_manager - await task_manager.close() + await task_manager.teardown() app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py index 0731009a380..970f8aeb67e 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py @@ -28,7 +28,7 @@ from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.docker_constants import SUFFIX_EGRESS_PROXY_NAME -from servicelib.fastapi.long_running_tasks.client import TaskId +from servicelib.long_running_tasks.models import TaskId from simcore_service_dynamic_sidecar._meta import API_VTAG from simcore_service_dynamic_sidecar.api.rest.containers import _INACTIVE_FOR_LONG_TIME from simcore_service_dynamic_sidecar.core.application import AppState From fd508108cbcd46adbe1b2ea7d80c0de7327fb758 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 09:56:07 +0200 Subject: [PATCH 31/46] fixed imports --- .../02/test_dynamic_sidecar_nodeports_integration.py | 8 ++++---- .../unit/test_api_rest_containers_long_running_tasks.py | 7 ++----- .../tests/unit/test_api_rest_prometheus_metrics.py | 7 ++----- .../tests/unit/test_api_rest_workflow_service_metrics.py | 7 ++----- 4 files changed, 10 insertions(+), 19 deletions(-) diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 3469914a677..ce9c08b5b5d 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -50,12 +50,12 @@ from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from servicelib.fastapi.long_running_tasks.client import ( - Client, +from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result +from servicelib.long_running_tasks.models import ( + ProgressMessage, + ProgressPercent, TaskId, - periodic_task_result, ) -from servicelib.long_running_tasks.models import ProgressMessage, ProgressPercent from servicelib.progress_bar import ProgressBarData from servicelib.sequences_utils import pairwise from settings_library.rabbit import RabbitSettings diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py index 75677e1d7f7..b00491ce764 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py @@ -28,12 +28,9 @@ from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict -from servicelib.fastapi.long_running_tasks.client import ( - Client, - TaskId, - periodic_task_result, -) +from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result from servicelib.fastapi.long_running_tasks.client import setup as client_setup +from servicelib.long_running_tasks.models import TaskId from simcore_sdk.node_ports_common.exceptions import NodeNotFound from simcore_service_dynamic_sidecar._meta import API_VTAG from simcore_service_dynamic_sidecar.api.rest import containers_long_running_tasks diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py index 7d4454b7e0a..78e5b22046e 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py @@ -17,12 +17,9 @@ from models_library.services_creation import CreateServiceMetricsAdditionalParams from pydantic import AnyHttpUrl, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict -from servicelib.fastapi.long_running_tasks.client import ( - Client, - TaskId, - periodic_task_result, -) +from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result from servicelib.fastapi.long_running_tasks.client import setup as client_setup +from servicelib.long_running_tasks.models import TaskId from simcore_service_dynamic_sidecar._meta import API_VTAG from simcore_service_dynamic_sidecar.models.schemas.containers import ( ContainersComposeSpec, diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py index b276f00cb2c..ee3ab015a1c 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py @@ -32,12 +32,9 @@ from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict -from servicelib.fastapi.long_running_tasks.client import ( - Client, - TaskId, - periodic_task_result, -) +from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result from servicelib.fastapi.long_running_tasks.client import setup as client_setup +from servicelib.long_running_tasks.models import TaskId from simcore_service_dynamic_sidecar._meta import API_VTAG from simcore_service_dynamic_sidecar.core.docker_utils import get_container_states from simcore_service_dynamic_sidecar.models.schemas.containers import ( From 08a65e79856a2819cd93bb310fdddc361baee072 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 10:36:19 +0200 Subject: [PATCH 32/46] pylint --- .../service-library/src/servicelib/long_running_tasks/task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 95240baf6ae..d4eb6fbda43 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -275,8 +275,9 @@ async def cancel_task( tracked_task = self._get_tracked_task(task_id, with_task_context) await self._cancel_tracked_task(tracked_task.task, task_id, reraise_errors=True) + @staticmethod async def _cancel_asyncio_task( - self, task: asyncio.Task, reference: str, *, reraise_errors: bool + task: asyncio.Task, reference: str, *, reraise_errors: bool ) -> None: task.cancel() with suppress(asyncio.CancelledError): From c30f80d3cd11013229e4a8d0c2b395cf70fa97cc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 10:52:30 +0200 Subject: [PATCH 33/46] removed some constants --- .../servicelib/aiohttp/long_running_tasks/client.py | 11 +++++++---- .../src/servicelib/long_running_tasks/constants.py | 2 -- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py index 34a2de06227..ed5675a457d 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py @@ -1,9 +1,11 @@ import asyncio import logging from collections.abc import AsyncGenerator -from typing import Any +from datetime import timedelta +from typing import Any, Final from aiohttp import ClientConnectionError, ClientSession +from pydantic import PositiveFloat from tenacity import TryAgain, retry from tenacity.asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log @@ -12,7 +14,7 @@ from tenacity.wait import wait_random_exponential from yarl import URL -from ...long_running_tasks.constants import DEFAULT_POLL_INTERVAL_S, HOUR +from ...long_running_tasks.constants import DEFAULT_POLL_INTERVAL_S from ...long_running_tasks.models import ( LRTask, RequestBody, @@ -26,6 +28,7 @@ _logger = logging.getLogger(__name__) +_DEFAULT_CLIENT_TIMEOUT_S: Final[PositiveFloat] = timedelta(hours=1).total_seconds() _DEFAULT_AIOHTTP_RETRY_POLICY: dict[str, Any] = { "retry": retry_if_exception_type(ClientConnectionError), @@ -49,7 +52,7 @@ async def _wait_for_completion( session: ClientSession, task_id: TaskId, status_url: URL, - client_timeout: int, + client_timeout: PositiveFloat, ) -> AsyncGenerator[TaskProgress, None]: try: async for attempt in AsyncRetrying( @@ -98,7 +101,7 @@ async def long_running_task_request( session: ClientSession, url: URL, json: RequestBody | None = None, - client_timeout: int = 1 * HOUR, + client_timeout: PositiveFloat = _DEFAULT_CLIENT_TIMEOUT_S, ) -> AsyncGenerator[LRTask, None]: """Will use the passed `ClientSession` to call an oSparc long running task `url` passing `json` as request body. diff --git a/packages/service-library/src/servicelib/long_running_tasks/constants.py b/packages/service-library/src/servicelib/long_running_tasks/constants.py index 45567e3e80b..b5e729665cc 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/constants.py +++ b/packages/service-library/src/servicelib/long_running_tasks/constants.py @@ -1,8 +1,6 @@ from datetime import timedelta from typing import Final -MINUTE: Final[int] = 60 # in secs -HOUR: Final[int] = 60 * MINUTE # in secs DEFAULT_POLL_INTERVAL_S: Final[float] = 1 DEFAULT_STALE_TASK_CHECK_INTERVAL: Final[timedelta] = timedelta(minutes=1) From 87b6b4533c38e3d9ae607841e13bd82ee8f43dd0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 13:22:18 +0200 Subject: [PATCH 34/46] moved module --- .../src/servicelib/aiohttp/long_running_tasks/_routes.py | 2 +- .../src/servicelib/fastapi/long_running_tasks/_routes.py | 2 +- .../long_running_tasks/{expose => }/endpoint_responses.py | 4 ++-- .../src/servicelib/long_running_tasks/expose/__init__.py | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename packages/service-library/src/servicelib/long_running_tasks/{expose => }/endpoint_responses.py (91%) delete mode 100644 packages/service-library/src/servicelib/long_running_tasks/expose/__init__.py diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index e5ab406af97..41d028c4189 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -5,7 +5,7 @@ from servicelib.aiohttp import status from servicelib.aiohttp.rest_responses import create_data_response -from ...long_running_tasks.expose import endpoint_responses +from ...long_running_tasks import endpoint_responses from ...long_running_tasks.models import TaskGet, TaskId, TaskStatus from ..requests_validation import parse_request_path_parameters_as from ._dependencies import get_task_context, get_tasks_manager diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index ea59ee095cb..70600395c23 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Depends, Request, status -from ...long_running_tasks.expose import endpoint_responses +from ...long_running_tasks import endpoint_responses from ...long_running_tasks.models import TaskGet, TaskId, TaskResult, TaskStatus from ...long_running_tasks.task import TasksManager from ..requests_decorators import cancel_on_disconnect diff --git a/packages/service-library/src/servicelib/long_running_tasks/expose/endpoint_responses.py b/packages/service-library/src/servicelib/long_running_tasks/endpoint_responses.py similarity index 91% rename from packages/service-library/src/servicelib/long_running_tasks/expose/endpoint_responses.py rename to packages/service-library/src/servicelib/long_running_tasks/endpoint_responses.py index 5599b906154..cf0222c6265 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/expose/endpoint_responses.py +++ b/packages/service-library/src/servicelib/long_running_tasks/endpoint_responses.py @@ -1,7 +1,7 @@ from typing import Any -from ..models import TaskGetWithoutHref, TaskId, TaskStatus -from ..task import TaskContext, TasksManager, TrackedTask +from .models import TaskGetWithoutHref, TaskId, TaskStatus +from .task import TaskContext, TasksManager, TrackedTask def list_tasks( diff --git a/packages/service-library/src/servicelib/long_running_tasks/expose/__init__.py b/packages/service-library/src/servicelib/long_running_tasks/expose/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 From 8d894efa20698f1edf7f8bae401619686042eb0b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 13:23:27 +0200 Subject: [PATCH 35/46] rename module --- .../servicelib/aiohttp/long_running_tasks/_routes.py | 10 +++++----- .../servicelib/fastapi/long_running_tasks/_routes.py | 10 +++++----- ...ndpoint_responses.py => http_endpoint_responses.py} | 0 3 files changed, 10 insertions(+), 10 deletions(-) rename packages/service-library/src/servicelib/long_running_tasks/{endpoint_responses.py => http_endpoint_responses.py} (100%) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 41d028c4189..f57e1b0408f 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -5,7 +5,7 @@ from servicelib.aiohttp import status from servicelib.aiohttp.rest_responses import create_data_response -from ...long_running_tasks import endpoint_responses +from ...long_running_tasks import http_endpoint_responses from ...long_running_tasks.models import TaskGet, TaskId, TaskStatus from ..requests_validation import parse_request_path_parameters_as from ._dependencies import get_task_context, get_tasks_manager @@ -30,7 +30,7 @@ async def list_tasks(request: web.Request) -> web.Response: result_href=f"{request.app.router['get_task_result'].url_for(task_id=t.task_id)}", abort_href=f"{request.app.router['cancel_and_delete_task'].url_for(task_id=t.task_id)}", ) - for t in endpoint_responses.list_tasks(tasks_manager, task_context) + for t in http_endpoint_responses.list_tasks(tasks_manager, task_context) ] ) @@ -41,7 +41,7 @@ async def get_task_status(request: web.Request) -> web.Response: tasks_manager = get_tasks_manager(request.app) task_context = get_task_context(request) - task_status: TaskStatus = endpoint_responses.get_task_status( + task_status: TaskStatus = http_endpoint_responses.get_task_status( tasks_manager, task_context, path_params.task_id ) return create_data_response(task_status) @@ -54,7 +54,7 @@ async def get_task_result(request: web.Request) -> web.Response | Any: task_context = get_task_context(request) # NOTE: this might raise an exception that will be catched by the _error_handlers - return await endpoint_responses.get_task_result( + return await http_endpoint_responses.get_task_result( tasks_manager, task_context, path_params.task_id ) @@ -64,7 +64,7 @@ async def cancel_and_delete_task(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(_PathParam, request) tasks_manager = get_tasks_manager(request.app) task_context = get_task_context(request) - await endpoint_responses.remove_task( + await http_endpoint_responses.remove_task( tasks_manager, task_context, path_params.task_id ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index 70600395c23..c1b8ca7ea29 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Depends, Request, status -from ...long_running_tasks import endpoint_responses +from ...long_running_tasks import http_endpoint_responses from ...long_running_tasks.models import TaskGet, TaskId, TaskResult, TaskStatus from ...long_running_tasks.task import TasksManager from ..requests_decorators import cancel_on_disconnect @@ -27,7 +27,7 @@ async def list_tasks( request.url_for("cancel_and_delete_task", task_id=t.task_id) ), ) - for t in endpoint_responses.list_tasks(tasks_manager, task_context=None) + for t in http_endpoint_responses.list_tasks(tasks_manager, task_context=None) ] @@ -45,7 +45,7 @@ async def get_task_status( tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], ) -> TaskStatus: assert request # nosec - return endpoint_responses.get_task_status( + return http_endpoint_responses.get_task_status( tasks_manager, task_context=None, task_id=task_id ) @@ -66,7 +66,7 @@ async def get_task_result( tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], ) -> TaskResult | Any: assert request # nosec - return await endpoint_responses.get_task_result( + return await http_endpoint_responses.get_task_result( tasks_manager, task_context=None, task_id=task_id ) @@ -87,6 +87,6 @@ async def cancel_and_delete_task( tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], ) -> None: assert request # nosec - await endpoint_responses.remove_task( + await http_endpoint_responses.remove_task( tasks_manager, task_context=None, task_id=task_id ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/endpoint_responses.py b/packages/service-library/src/servicelib/long_running_tasks/http_endpoint_responses.py similarity index 100% rename from packages/service-library/src/servicelib/long_running_tasks/endpoint_responses.py rename to packages/service-library/src/servicelib/long_running_tasks/http_endpoint_responses.py From d2dbc95f5777c227c27c58174147278a93c9bab7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 13:38:00 +0200 Subject: [PATCH 36/46] import from correct place --- .../web/server/src/simcore_service_webserver/tasks/_rest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/tasks/_rest.py b/services/web/server/src/simcore_service_webserver/tasks/_rest.py index 6592af29784..e1a73e1a6ac 100644 --- a/services/web/server/src/simcore_service_webserver/tasks/_rest.py +++ b/services/web/server/src/simcore_service_webserver/tasks/_rest.py @@ -29,6 +29,7 @@ parse_request_path_parameters_as, ) from servicelib.aiohttp.rest_responses import create_data_response +from servicelib.long_running_tasks import http_endpoint_responses from servicelib.rabbitmq.rpc_interfaces.async_jobs import async_jobs from .._meta import API_VTAG @@ -58,7 +59,9 @@ async def get_async_jobs(request: web.Request) -> web.Response: inprocess_task_manager = get_tasks_manager(request.app) inprocess_task_context = get_task_context(request) - inprocess_tracked_tasks = inprocess_task_manager.list_tasks(inprocess_task_context) + inprocess_tracked_tasks = http_endpoint_responses.list_tasks( + inprocess_task_manager, inprocess_task_context + ) _req_ctx = AuthenticatedRequestContext.model_validate(request) From 8462647dc4f195239855fa6149350c5ef4bd0d86 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Jun 2025 15:50:32 +0200 Subject: [PATCH 37/46] refactor internals --- .../aiohttp/long_running_tasks/_dependencies.py | 12 +----------- .../servicelib/aiohttp/long_running_tasks/_routes.py | 8 +++++++- .../servicelib/aiohttp/long_running_tasks/server.py | 6 ++---- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py index b6b9a89d26a..b3bae57fe14 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py @@ -1,12 +1,7 @@ -from typing import Any - from aiohttp import web from ...long_running_tasks.task import TasksManager -from ._constants import ( - APP_LONG_RUNNING_TASKS_MANAGER_KEY, - RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, -) +from ._constants import APP_LONG_RUNNING_TASKS_MANAGER_KEY # NOTE: figure out how to remove these and expose them differently if possible @@ -14,8 +9,3 @@ def get_tasks_manager(app: web.Application) -> TasksManager: output: TasksManager = app[APP_LONG_RUNNING_TASKS_MANAGER_KEY] return output - - -def get_task_context(request: web.Request) -> dict[str, Any]: - output: dict[str, Any] = request[RQT_LONG_RUNNING_TASKS_CONTEXT_KEY] - return output diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index f57e1b0408f..20c5cbb2fb9 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -8,11 +8,17 @@ from ...long_running_tasks import http_endpoint_responses from ...long_running_tasks.models import TaskGet, TaskId, TaskStatus from ..requests_validation import parse_request_path_parameters_as -from ._dependencies import get_task_context, get_tasks_manager +from ._constants import RQT_LONG_RUNNING_TASKS_CONTEXT_KEY +from ._dependencies import get_tasks_manager routes = web.RouteTableDef() +def get_task_context(request: web.Request) -> dict[str, Any]: + output: dict[str, Any] = request[RQT_LONG_RUNNING_TASKS_CONTEXT_KEY] + return output + + class _PathParam(BaseModel): task_id: TaskId diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py index 7ec90cb069e..8b19cf7e7e4 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py @@ -6,10 +6,8 @@ running task. """ -from ._dependencies import ( - get_task_context, - get_tasks_manager, -) +from ._dependencies import get_tasks_manager +from ._routes import get_task_context from ._server import setup, start_long_running_task __all__: tuple[str, ...] = ( From cb5064bf3171d971d2ea9026d3fb04d0ad88186d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 12 Jun 2025 08:30:26 +0200 Subject: [PATCH 38/46] refactor --- .../src/servicelib/long_running_tasks/task.py | 63 +++++++++---------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index d4eb6fbda43..c89e94ce476 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -30,35 +30,39 @@ seconds=1 ).total_seconds() +TrackedTaskGroupDict: TypeAlias = dict[TaskId, TrackedTask] +TaskContext: TypeAlias = dict[str, Any] + async def _await_task(task: asyncio.Task) -> None: await task -def _mark_task_to_remove_if_required( - task_id: TaskId, - tasks_to_remove: list[TaskId], - tracked_task: TrackedTask, - utc_now: datetime.datetime, - stale_timeout_s: float, -) -> None: - if tracked_task.fire_and_forget: - return - - if tracked_task.last_status_check is None: - # the task just added or never received a poll request - elapsed_from_start = (utc_now - tracked_task.started).seconds - if elapsed_from_start > stale_timeout_s: - tasks_to_remove.append(task_id) - else: - # the task status was already queried by the client - elapsed_from_last_poll = (utc_now - tracked_task.last_status_check).seconds - if elapsed_from_last_poll > stale_timeout_s: - tasks_to_remove.append(task_id) +def _get_tasks_to_remove( + tasks_groups: dict[TaskName, TrackedTaskGroupDict], + stale_task_detect_timeout_s: PositiveFloat, +) -> list[TaskId]: + utc_now = datetime.datetime.now(tz=datetime.UTC) + tasks_to_remove: list[TaskId] = [] + for tasks in tasks_groups.values(): + for task_id, tracked_task in tasks.items(): + if tracked_task.fire_and_forget: + continue -TrackedTaskGroupDict: TypeAlias = dict[TaskId, TrackedTask] -TaskContext: TypeAlias = dict[str, Any] + if tracked_task.last_status_check is None: + # the task just added or never received a poll request + elapsed_from_start = (utc_now - tracked_task.started).seconds + if elapsed_from_start > stale_task_detect_timeout_s: + tasks_to_remove.append(task_id) + else: + # the task status was already queried by the client + elapsed_from_last_poll = ( + utc_now - tracked_task.last_status_check + ).seconds + if elapsed_from_last_poll > stale_task_detect_timeout_s: + tasks_to_remove.append(task_id) + return tasks_to_remove class TasksManager: @@ -125,18 +129,9 @@ async def _stale_tasks_monitor_worker(self) -> None: # Since we own the client, we assume (for now) this # will not be the case. - utc_now = datetime.datetime.now(tz=datetime.UTC) - - tasks_to_remove: list[TaskId] = [] - for tasks in self._tasks_groups.values(): - for task_id, tracked_task in tasks.items(): - _mark_task_to_remove_if_required( - task_id, - tasks_to_remove, - tracked_task, - utc_now, - self.stale_task_detect_timeout_s, - ) + tasks_to_remove = _get_tasks_to_remove( + self._tasks_groups, self.stale_task_detect_timeout_s + ) # finally remove tasks and warn for task_id in tasks_to_remove: From d0d8f8553f24befb76e028e2919ba805a17c7994 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 12 Jun 2025 08:37:19 +0200 Subject: [PATCH 39/46] using long_running_manager for aiohttp --- .../api_schemas_long_running_tasks/tasks.py | 4 +- .../aiohttp/long_running_tasks/_constants.py | 12 +++--- .../long_running_tasks/_dependencies.py | 11 +++-- .../aiohttp/long_running_tasks/_manager.py | 41 +++++++++++++++++++ .../aiohttp/long_running_tasks/_routes.py | 40 +++++++++--------- .../aiohttp/long_running_tasks/_server.py | 33 +++++++-------- .../aiohttp/long_running_tasks/server.py | 6 +-- .../http_endpoint_responses.py | 9 ++-- .../servicelib/long_running_tasks/models.py | 4 +- .../simcore_service_webserver/tasks/_rest.py | 9 ++-- 10 files changed, 99 insertions(+), 70 deletions(-) create mode 100644 packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py diff --git a/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py b/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py index 496151b36ed..392d8841451 100644 --- a/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py +++ b/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py @@ -18,12 +18,12 @@ class TaskResult(BaseModel): error: Any | None -class TaskGetWithoutHref(BaseModel): +class TaskBase(BaseModel): task_id: TaskId task_name: str -class TaskGet(TaskGetWithoutHref): +class TaskGet(TaskBase): status_href: str result_href: str abort_href: str diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_constants.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_constants.py index 79594cb18b8..fe38782e9ff 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_constants.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_constants.py @@ -3,9 +3,9 @@ from pydantic import PositiveFloat MINUTE: Final[PositiveFloat] = 60 -APP_LONG_RUNNING_TASKS_MANAGER_KEY: Final[ - str -] = f"{__name__ }.long_running_tasks.tasks_manager" -RQT_LONG_RUNNING_TASKS_CONTEXT_KEY: Final[ - str -] = f"{__name__}.long_running_tasks.context" +APP_LONG_RUNNING_MANAGER_KEY: Final[str] = ( + f"{__name__ }.long_running_tasks.tasks_manager" +) +RQT_LONG_RUNNING_TASKS_CONTEXT_KEY: Final[str] = ( + f"{__name__}.long_running_tasks.context" +) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py index b3bae57fe14..0ccfd3c6a40 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py @@ -1,11 +1,10 @@ -from aiohttp import web +from typing import Any -from ...long_running_tasks.task import TasksManager -from ._constants import APP_LONG_RUNNING_TASKS_MANAGER_KEY +from aiohttp import web -# NOTE: figure out how to remove these and expose them differently if possible +from ._constants import RQT_LONG_RUNNING_TASKS_CONTEXT_KEY -def get_tasks_manager(app: web.Application) -> TasksManager: - output: TasksManager = app[APP_LONG_RUNNING_TASKS_MANAGER_KEY] +def get_task_context(request: web.Request) -> dict[str, Any]: + output: dict[str, Any] = request[RQT_LONG_RUNNING_TASKS_CONTEXT_KEY] return output diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py new file mode 100644 index 00000000000..e934aded404 --- /dev/null +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py @@ -0,0 +1,41 @@ +import datetime + +from aiohttp import web + +from ...long_running_tasks.base_long_running_manager import BaseLongRunningManager +from ...long_running_tasks.task import TaskContext, TasksManager +from ._constants import APP_LONG_RUNNING_MANAGER_KEY +from ._dependencies import get_task_context + + +class AiohttpLongRunningManager(BaseLongRunningManager): + def __init__( + self, + app: web.Application, + stale_task_check_interval: datetime.timedelta, + stale_task_detect_timeout: datetime.timedelta, + ): + self._app = app + self._tasks_manager = TasksManager( + stale_task_check_interval=stale_task_check_interval, + stale_task_detect_timeout=stale_task_detect_timeout, + ) + + @property + def tasks_manager(self) -> TasksManager: + return self._tasks_manager + + async def setup(self) -> None: + await self._tasks_manager.setup() + + async def teardown(self) -> None: + await self._tasks_manager.teardown() + + @staticmethod + def get_task_context(request: web.Request) -> TaskContext: + return get_task_context(request) + + +def get_long_running_manager(app: web.Application) -> AiohttpLongRunningManager: + output: AiohttpLongRunningManager = app[APP_LONG_RUNNING_MANAGER_KEY] + return output diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 20c5cbb2fb9..513203f6a1e 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -3,30 +3,23 @@ from aiohttp import web from pydantic import BaseModel from servicelib.aiohttp import status -from servicelib.aiohttp.rest_responses import create_data_response from ...long_running_tasks import http_endpoint_responses from ...long_running_tasks.models import TaskGet, TaskId, TaskStatus from ..requests_validation import parse_request_path_parameters_as -from ._constants import RQT_LONG_RUNNING_TASKS_CONTEXT_KEY -from ._dependencies import get_tasks_manager +from ..rest_responses import create_data_response +from ._manager import get_long_running_manager routes = web.RouteTableDef() -def get_task_context(request: web.Request) -> dict[str, Any]: - output: dict[str, Any] = request[RQT_LONG_RUNNING_TASKS_CONTEXT_KEY] - return output - - class _PathParam(BaseModel): task_id: TaskId @routes.get("", name="list_tasks") async def list_tasks(request: web.Request) -> web.Response: - tasks_manager = get_tasks_manager(request.app) - task_context = get_task_context(request) + long_running_manager = get_long_running_manager(request.app) return create_data_response( [ TaskGet( @@ -36,7 +29,10 @@ async def list_tasks(request: web.Request) -> web.Response: result_href=f"{request.app.router['get_task_result'].url_for(task_id=t.task_id)}", abort_href=f"{request.app.router['cancel_and_delete_task'].url_for(task_id=t.task_id)}", ) - for t in http_endpoint_responses.list_tasks(tasks_manager, task_context) + for t in http_endpoint_responses.list_tasks( + long_running_manager.tasks_manager, + long_running_manager.get_task_context(request), + ) ] ) @@ -44,11 +40,12 @@ async def list_tasks(request: web.Request) -> web.Response: @routes.get("/{task_id}", name="get_task_status") async def get_task_status(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(_PathParam, request) - tasks_manager = get_tasks_manager(request.app) - task_context = get_task_context(request) + long_running_manager = get_long_running_manager(request.app) task_status: TaskStatus = http_endpoint_responses.get_task_status( - tasks_manager, task_context, path_params.task_id + long_running_manager.tasks_manager, + long_running_manager.get_task_context(request), + path_params.task_id, ) return create_data_response(task_status) @@ -56,21 +53,24 @@ async def get_task_status(request: web.Request) -> web.Response: @routes.get("/{task_id}/result", name="get_task_result") async def get_task_result(request: web.Request) -> web.Response | Any: path_params = parse_request_path_parameters_as(_PathParam, request) - tasks_manager = get_tasks_manager(request.app) - task_context = get_task_context(request) + long_running_manager = get_long_running_manager(request.app) # NOTE: this might raise an exception that will be catched by the _error_handlers return await http_endpoint_responses.get_task_result( - tasks_manager, task_context, path_params.task_id + long_running_manager.tasks_manager, + long_running_manager.get_task_context(request), + path_params.task_id, ) @routes.delete("/{task_id}", name="cancel_and_delete_task") async def cancel_and_delete_task(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(_PathParam, request) - tasks_manager = get_tasks_manager(request.app) - task_context = get_task_context(request) + long_running_manager = get_long_running_manager(request.app) + await http_endpoint_responses.remove_task( - tasks_manager, task_context, path_params.task_id + long_running_manager.tasks_manager, + long_running_manager.get_task_context(request), + path_params.task_id, ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index da33ecd3562..a68bb4a67b0 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -1,6 +1,5 @@ import asyncio import datetime -import logging from collections.abc import AsyncGenerator, Callable from functools import wraps from typing import Any @@ -15,22 +14,15 @@ DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) from ...long_running_tasks.models import TaskGet -from ...long_running_tasks.task import ( - TaskContext, - TaskProtocol, - TasksManager, - start_task, -) +from ...long_running_tasks.task import TaskContext, TaskProtocol, start_task from ..typing_extension import Handler from . import _routes from ._constants import ( - APP_LONG_RUNNING_TASKS_MANAGER_KEY, + APP_LONG_RUNNING_MANAGER_KEY, RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, ) -from ._dependencies import get_tasks_manager from ._error_handlers import base_long_running_error_handler - -_logger = logging.getLogger(__name__) +from ._manager import AiohttpLongRunningManager, get_long_running_manager def no_ops_decorator(handler: Handler): @@ -59,12 +51,12 @@ async def start_long_running_task( task_context: TaskContext, **task_kwargs: Any, ) -> web.Response: - task_manager = get_tasks_manager(request_.app) + long_running_manager = get_long_running_manager(request_.app) task_name = _create_task_name_from_request(request_) task_id = None try: task_id = start_task( - task_manager, + long_running_manager.tasks_manager, task_, fire_and_forget=fire_and_forget, task_context=task_context, @@ -99,8 +91,10 @@ async def start_long_running_task( except asyncio.CancelledError: # cancel the task, the client has disconnected if task_id: - task_manager = get_tasks_manager(request_.app) - await task_manager.cancel_task(task_id, with_task_context=None) + long_running_manager = get_long_running_manager(request_.app) + await long_running_manager.tasks_manager.cancel_task( + task_id, with_task_context=None + ) raise @@ -147,19 +141,20 @@ async def on_cleanup_ctx(app: web.Application) -> AsyncGenerator[None, None]: app.middlewares.append(base_long_running_error_handler) # add components to state - app[APP_LONG_RUNNING_TASKS_MANAGER_KEY] = long_running_task_manager = ( - TasksManager( + app[APP_LONG_RUNNING_MANAGER_KEY] = long_running_manager = ( + AiohttpLongRunningManager( + app=app, stale_task_check_interval=stale_task_check_interval, stale_task_detect_timeout=stale_task_detect_timeout, ) ) - await long_running_task_manager.setup() + await long_running_manager.setup() yield # cleanup - await long_running_task_manager.teardown() + await long_running_manager.teardown() # add routing (done at setup-time) _wrap_and_add_routes( diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py index 8b19cf7e7e4..c5de3f1d5e1 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py @@ -6,13 +6,11 @@ running task. """ -from ._dependencies import get_tasks_manager -from ._routes import get_task_context +from ._manager import get_long_running_manager from ._server import setup, start_long_running_task __all__: tuple[str, ...] = ( - "get_task_context", - "get_tasks_manager", + "get_long_running_manager", "setup", "start_long_running_task", ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/http_endpoint_responses.py b/packages/service-library/src/servicelib/long_running_tasks/http_endpoint_responses.py index cf0222c6265..be873f1a1a2 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/http_endpoint_responses.py +++ b/packages/service-library/src/servicelib/long_running_tasks/http_endpoint_responses.py @@ -1,19 +1,16 @@ from typing import Any -from .models import TaskGetWithoutHref, TaskId, TaskStatus +from .models import TaskBase, TaskId, TaskStatus from .task import TaskContext, TasksManager, TrackedTask def list_tasks( tasks_manager: TasksManager, task_context: TaskContext | None -) -> list[TaskGetWithoutHref]: +) -> list[TaskBase]: tracked_tasks: list[TrackedTask] = tasks_manager.list_tasks( with_task_context=task_context ) - return [ - TaskGetWithoutHref(task_id=t.task_id, task_name=t.task_name) - for t in tracked_tasks - ] + return [TaskBase(task_id=t.task_id, task_name=t.task_name) for t in tracked_tasks] def get_task_status( diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index deb3d0e5d6e..37fc968568d 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -12,8 +12,8 @@ TaskProgress, ) from models_library.api_schemas_long_running_tasks.tasks import ( + TaskBase, TaskGet, - TaskGetWithoutHref, TaskResult, TaskStatus, ) @@ -79,7 +79,7 @@ async def result(self) -> Any: "ProgressMessage", "ProgressPercent", "TaskGet", - "TaskGetWithoutHref", + "TaskBase", "TaskId", "TaskProgress", "TaskResult", diff --git a/services/web/server/src/simcore_service_webserver/tasks/_rest.py b/services/web/server/src/simcore_service_webserver/tasks/_rest.py index e1a73e1a6ac..9b6ce8411d8 100644 --- a/services/web/server/src/simcore_service_webserver/tasks/_rest.py +++ b/services/web/server/src/simcore_service_webserver/tasks/_rest.py @@ -22,8 +22,7 @@ from pydantic import BaseModel from servicelib.aiohttp import status from servicelib.aiohttp.long_running_tasks.server import ( - get_task_context, - get_tasks_manager, + get_long_running_manager, ) from servicelib.aiohttp.requests_validation import ( parse_request_path_parameters_as, @@ -57,10 +56,10 @@ @handle_export_data_exceptions @webserver_request_context_decorator async def get_async_jobs(request: web.Request) -> web.Response: - inprocess_task_manager = get_tasks_manager(request.app) - inprocess_task_context = get_task_context(request) + inprocess_long_running_manager = get_long_running_manager(request.app) inprocess_tracked_tasks = http_endpoint_responses.list_tasks( - inprocess_task_manager, inprocess_task_context + inprocess_long_running_manager.tasks_manager, + inprocess_long_running_manager.get_task_context(request), ) _req_ctx = AuthenticatedRequestContext.model_validate(request) From 01210d570a47b5c53b3df84b98c7c8ded2813103 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 12 Jun 2025 08:41:14 +0200 Subject: [PATCH 40/46] refactor internals --- .../src/servicelib/aiohttp/long_running_tasks/_manager.py | 2 +- .../long_running_tasks/{_dependencies.py => _request.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename packages/service-library/src/servicelib/aiohttp/long_running_tasks/{_dependencies.py => _request.py} (100%) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py index e934aded404..aa1fc4ea09e 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py @@ -5,7 +5,7 @@ from ...long_running_tasks.base_long_running_manager import BaseLongRunningManager from ...long_running_tasks.task import TaskContext, TasksManager from ._constants import APP_LONG_RUNNING_MANAGER_KEY -from ._dependencies import get_task_context +from ._request import get_task_context class AiohttpLongRunningManager(BaseLongRunningManager): diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_request.py similarity index 100% rename from packages/service-library/src/servicelib/aiohttp/long_running_tasks/_dependencies.py rename to packages/service-library/src/servicelib/aiohttp/long_running_tasks/_request.py From aa038484d65b6e0ca2bf8b92576c96d7d08151a1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 12 Jun 2025 08:54:28 +0200 Subject: [PATCH 41/46] fastapi base migrated to long_running_manager --- .../long_running_tasks/_dependencies.py | 10 +++--- .../fastapi/long_running_tasks/_manager.py | 30 ++++++++++++++++++ .../fastapi/long_running_tasks/_routes.py | 31 +++++++++++++------ .../fastapi/long_running_tasks/_server.py | 16 ++++++---- .../fastapi/long_running_tasks/server.py | 4 +-- .../base_long_running_manager.py | 22 +++++++++++++ 6 files changed, 91 insertions(+), 22 deletions(-) create mode 100644 packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py create mode 100644 packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_dependencies.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_dependencies.py index e2c2fdc4b00..ced9efa4f16 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_dependencies.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_dependencies.py @@ -1,8 +1,10 @@ from fastapi import Request -from ...long_running_tasks.task import TasksManager +from ._manager import FastAPILongRunningManager -def get_tasks_manager(request: Request) -> TasksManager: - output: TasksManager = request.app.state.long_running_task_manager - return output +def get_long_running_manager(request: Request) -> FastAPILongRunningManager: + assert isinstance( + request.app.state.long_running_manager, FastAPILongRunningManager + ) # nosec + return request.app.state.long_running_manager diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py new file mode 100644 index 00000000000..bbc3e098a7e --- /dev/null +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py @@ -0,0 +1,30 @@ +import datetime + +from fastapi import FastAPI + +from ...long_running_tasks.base_long_running_manager import BaseLongRunningManager +from ...long_running_tasks.task import TasksManager + + +class FastAPILongRunningManager(BaseLongRunningManager): + def __init__( + self, + app: FastAPI, + stale_task_check_interval: datetime.timedelta, + stale_task_detect_timeout: datetime.timedelta, + ): + self._app = app + self._tasks_manager = TasksManager( + stale_task_check_interval=stale_task_check_interval, + stale_task_detect_timeout=stale_task_detect_timeout, + ) + + @property + def tasks_manager(self) -> TasksManager: + return self._tasks_manager + + async def setup(self) -> None: + await self._tasks_manager.setup() + + async def teardown(self) -> None: + await self._tasks_manager.teardown() diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index c1b8ca7ea29..8b474c8add9 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -4,9 +4,9 @@ from ...long_running_tasks import http_endpoint_responses from ...long_running_tasks.models import TaskGet, TaskId, TaskResult, TaskStatus -from ...long_running_tasks.task import TasksManager from ..requests_decorators import cancel_on_disconnect -from ._dependencies import get_tasks_manager +from ._dependencies import get_long_running_manager +from ._manager import FastAPILongRunningManager router = APIRouter(prefix="/task") @@ -14,7 +14,10 @@ @router.get("", response_model=list[TaskGet]) @cancel_on_disconnect async def list_tasks( - request: Request, tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)] + request: Request, + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], ) -> list[TaskGet]: assert request # nosec return [ @@ -27,7 +30,9 @@ async def list_tasks( request.url_for("cancel_and_delete_task", task_id=t.task_id) ), ) - for t in http_endpoint_responses.list_tasks(tasks_manager, task_context=None) + for t in http_endpoint_responses.list_tasks( + long_running_manager.tasks_manager, task_context=None + ) ] @@ -42,11 +47,13 @@ async def list_tasks( async def get_task_status( request: Request, task_id: TaskId, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], ) -> TaskStatus: assert request # nosec return http_endpoint_responses.get_task_status( - tasks_manager, task_context=None, task_id=task_id + long_running_manager.tasks_manager, task_context=None, task_id=task_id ) @@ -63,11 +70,13 @@ async def get_task_status( async def get_task_result( request: Request, task_id: TaskId, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], ) -> TaskResult | Any: assert request # nosec return await http_endpoint_responses.get_task_result( - tasks_manager, task_context=None, task_id=task_id + long_running_manager.tasks_manager, task_context=None, task_id=task_id ) @@ -84,9 +93,11 @@ async def get_task_result( async def cancel_and_delete_task( request: Request, task_id: TaskId, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], ) -> None: assert request # nosec await http_endpoint_responses.remove_task( - tasks_manager, task_context=None, task_id=task_id + long_running_manager.tasks_manager, task_context=None, task_id=task_id ) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index 6afa935b013..272250ae258 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -7,8 +7,8 @@ DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) from ...long_running_tasks.errors import BaseLongRunningError -from ...long_running_tasks.task import TasksManager from ._error_handlers import base_long_running_error_handler +from ._manager import FastAPILongRunningManager from ._routes import router @@ -36,14 +36,18 @@ async def on_startup() -> None: app.include_router(main_router) # add components to state - app.state.long_running_task_manager = TasksManager( - stale_task_check_interval=stale_task_check_interval, - stale_task_detect_timeout=stale_task_detect_timeout, + app.state.long_running_manager = long_running_manager = ( + FastAPILongRunningManager( + app=app, + stale_task_check_interval=stale_task_check_interval, + stale_task_detect_timeout=stale_task_detect_timeout, + ) ) + await long_running_manager.setup() async def on_shutdown() -> None: - if app.state.long_running_task_manager: - task_manager: TasksManager = app.state.long_running_task_manager + if app.state.long_running_manager: + task_manager: FastAPILongRunningManager = app.state.long_running_manager await task_manager.teardown() app.add_event_handler("startup", on_startup) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py index 51240434a77..b7cf0fba60a 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/server.py @@ -6,11 +6,11 @@ running task. The client will take care of recovering the result from it. """ -from ._dependencies import get_tasks_manager +from ._dependencies import get_long_running_manager from ._server import setup __all__: tuple[str, ...] = ( - "get_tasks_manager", + "get_long_running_manager", "setup", ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py new file mode 100644 index 00000000000..d09428d3aa2 --- /dev/null +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -0,0 +1,22 @@ +from abc import ABC, abstractmethod + +from .task import TasksManager + + +class BaseLongRunningManager(ABC): + """ + Provides a commond inteface for aiohttp and fastapi services + """ + + @property + @abstractmethod + def tasks_manager(self) -> TasksManager: + pass + + @abstractmethod + async def setup(self) -> None: + pass + + @abstractmethod + async def teardown(self) -> None: + pass From edea03d30979e555faabd3b2d69c6cad5e2e7b03 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 12 Jun 2025 08:54:46 +0200 Subject: [PATCH 42/46] fixed tests --- .../test_long_running_tasks.py | 11 +++++++---- ...test_long_running_tasks_context_manager.py | 19 +++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index 7a5b919c34e..07809f928b0 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -19,9 +19,10 @@ from fastapi import APIRouter, Depends, FastAPI, status from httpx import AsyncClient from pydantic import TypeAdapter +from servicelib.fastapi.long_running_tasks._manager import FastAPILongRunningManager from servicelib.fastapi.long_running_tasks.client import setup as setup_client from servicelib.fastapi.long_running_tasks.server import ( - get_tasks_manager, + get_long_running_manager, ) from servicelib.fastapi.long_running_tasks.server import setup as setup_server from servicelib.long_running_tasks.models import ( @@ -30,7 +31,7 @@ TaskProgress, TaskStatus, ) -from servicelib.long_running_tasks.task import TaskContext, TasksManager, start_task +from servicelib.long_running_tasks.task import TaskContext, start_task from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay @@ -69,10 +70,12 @@ async def create_string_list_task( num_strings: int, sleep_time: float, fail: bool = False, - task_manager: TasksManager = Depends(get_tasks_manager), + long_running_manager: FastAPILongRunningManager = Depends( + get_long_running_manager + ), ) -> TaskId: task_id = start_task( - task_manager, + long_running_manager.tasks_manager, _string_list_task, num_strings=num_strings, sleep_time=sleep_time, diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index cf9890a0ed4..bcebb00f778 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -11,9 +11,10 @@ from httpx import AsyncClient from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter from servicelib.fastapi.long_running_tasks._context_manager import _ProgressManager +from servicelib.fastapi.long_running_tasks._manager import FastAPILongRunningManager from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result from servicelib.fastapi.long_running_tasks.client import setup as setup_client -from servicelib.fastapi.long_running_tasks.server import get_tasks_manager +from servicelib.fastapi.long_running_tasks.server import get_long_running_manager from servicelib.fastapi.long_running_tasks.server import setup as setup_server from servicelib.long_running_tasks.errors import ( TaskClientTimeoutError, @@ -24,7 +25,7 @@ TaskId, TaskProgress, ) -from servicelib.long_running_tasks.task import TasksManager, start_task +from servicelib.long_running_tasks.task import start_task TASK_SLEEP_INTERVAL: Final[PositiveFloat] = 0.1 @@ -55,16 +56,22 @@ def user_routes() -> APIRouter: @router.get("/api/success", status_code=status.HTTP_200_OK) async def create_task_user_defined_route( - tasks_manager: TasksManager = Depends(get_tasks_manager), + long_running_manager: FastAPILongRunningManager = Depends( + get_long_running_manager + ), ) -> TaskId: - task_id = start_task(tasks_manager, task=a_test_task) + task_id = start_task(long_running_manager.tasks_manager, task=a_test_task) return task_id @router.get("/api/failing", status_code=status.HTTP_200_OK) async def create_task_which_fails( - task_manager: TasksManager = Depends(get_tasks_manager), + long_running_manager: FastAPILongRunningManager = Depends( + get_long_running_manager + ), ) -> TaskId: - task_id = start_task(task_manager, task=a_failing_test_task) + task_id = start_task( + long_running_manager.tasks_manager, task=a_failing_test_task + ) return task_id return router From 19756ba9ba5fc343dcbeaa2c896b5b0d258bca0f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 12 Jun 2025 08:56:06 +0200 Subject: [PATCH 43/46] using annotations --- .../test_long_running_tasks_context_manager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index bcebb00f778..ddc735ca2e8 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -3,7 +3,7 @@ import asyncio from collections.abc import AsyncIterable -from typing import Final +from typing import Annotated, Final import pytest from asgi_lifespan import LifespanManager @@ -56,18 +56,18 @@ def user_routes() -> APIRouter: @router.get("/api/success", status_code=status.HTTP_200_OK) async def create_task_user_defined_route( - long_running_manager: FastAPILongRunningManager = Depends( - get_long_running_manager - ), + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], ) -> TaskId: task_id = start_task(long_running_manager.tasks_manager, task=a_test_task) return task_id @router.get("/api/failing", status_code=status.HTTP_200_OK) async def create_task_which_fails( - long_running_manager: FastAPILongRunningManager = Depends( - get_long_running_manager - ), + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], ) -> TaskId: task_id = start_task( long_running_manager.tasks_manager, task=a_failing_test_task From 66ec8e6bf0aedcb42f11c45fe26f354860c94aea Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 12 Jun 2025 08:58:24 +0200 Subject: [PATCH 44/46] using annotations --- .../test_long_running_tasks_context_manager.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index ddc735ca2e8..9072a70ddc0 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -60,8 +60,7 @@ async def create_task_user_defined_route( FastAPILongRunningManager, Depends(get_long_running_manager) ], ) -> TaskId: - task_id = start_task(long_running_manager.tasks_manager, task=a_test_task) - return task_id + return start_task(long_running_manager.tasks_manager, task=a_test_task) @router.get("/api/failing", status_code=status.HTTP_200_OK) async def create_task_which_fails( @@ -69,10 +68,7 @@ async def create_task_which_fails( FastAPILongRunningManager, Depends(get_long_running_manager) ], ) -> TaskId: - task_id = start_task( - long_running_manager.tasks_manager, task=a_failing_test_task - ) - return task_id + return start_task(long_running_manager.tasks_manager, task=a_failing_test_task) return router From b1d46bcd4664744f607f3ee28af29123780d2496 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 12 Jun 2025 08:58:40 +0200 Subject: [PATCH 45/46] refactor servies to use proper interface --- .../api/routes/dynamic_scheduler.py | 29 +++++---- .../api/rest/containers_long_running_tasks.py | 59 ++++++++++++------- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py index 4f01e803dd0..322ee491051 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py @@ -4,7 +4,8 @@ from fastapi import APIRouter, Depends, HTTPException, status from models_library.projects_nodes_io import NodeID from pydantic import BaseModel, PositiveInt -from servicelib.fastapi.long_running_tasks.server import get_tasks_manager +from servicelib.fastapi.long_running_tasks._manager import FastAPILongRunningManager +from servicelib.fastapi.long_running_tasks.server import get_long_running_manager from servicelib.long_running_tasks.errors import TaskAlreadyRunningError from servicelib.long_running_tasks.models import ( ProgressMessage, @@ -12,7 +13,7 @@ TaskId, TaskProgress, ) -from servicelib.long_running_tasks.task import TasksManager, start_task +from servicelib.long_running_tasks.task import start_task from tenacity import retry from tenacity.before_sleep import before_sleep_log from tenacity.retry import retry_if_result @@ -91,7 +92,9 @@ async def update_service_observation( ) async def delete_service_containers( node_uuid: NodeID, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], dynamic_sidecars_scheduler: Annotated[ DynamicSidecarsScheduler, Depends(get_dynamic_sidecar_scheduler) ], @@ -110,7 +113,7 @@ async def _progress_callback( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=_task_remove_service_containers, # type: ignore[arg-type] unique=True, node_uuid=node_uuid, @@ -149,7 +152,9 @@ async def get_service_state( ) async def save_service_state( node_uuid: NodeID, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], dynamic_sidecars_scheduler: Annotated[ DynamicSidecarsScheduler, Depends(get_dynamic_sidecar_scheduler) ], @@ -169,7 +174,7 @@ async def _progress_callback( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=_task_save_service_state, # type: ignore[arg-type] unique=True, node_uuid=node_uuid, @@ -191,7 +196,9 @@ async def _progress_callback( ) async def push_service_outputs( node_uuid: NodeID, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], dynamic_sidecars_scheduler: Annotated[ DynamicSidecarsScheduler, Depends(get_dynamic_sidecar_scheduler) ], @@ -210,7 +217,7 @@ async def _progress_callback( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=_task_push_service_outputs, # type: ignore[arg-type] unique=True, node_uuid=node_uuid, @@ -232,7 +239,9 @@ async def _progress_callback( ) async def delete_service_docker_resources( node_uuid: NodeID, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], dynamic_sidecars_scheduler: Annotated[ DynamicSidecarsScheduler, Depends(get_dynamic_sidecar_scheduler) ], @@ -246,7 +255,7 @@ async def _task_cleanup_service_docker_resources( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=_task_cleanup_service_docker_resources, # type: ignore[arg-type] unique=True, node_uuid=node_uuid, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py index da487ce392d..17ae8d9187f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py @@ -2,11 +2,12 @@ from typing import Annotated, cast from fastapi import APIRouter, Depends, FastAPI, Request, status -from servicelib.fastapi.long_running_tasks.server import get_tasks_manager +from servicelib.fastapi.long_running_tasks._manager import FastAPILongRunningManager +from servicelib.fastapi.long_running_tasks.server import get_long_running_manager from servicelib.fastapi.requests_decorators import cancel_on_disconnect from servicelib.long_running_tasks.errors import TaskAlreadyRunningError from servicelib.long_running_tasks.models import TaskId -from servicelib.long_running_tasks.task import TasksManager, start_task +from servicelib.long_running_tasks.task import start_task from ...core.settings import ApplicationSettings from ...models.schemas.application_health import ApplicationHealth @@ -48,7 +49,9 @@ @cancel_on_disconnect async def pull_user_servcices_docker_images( request: Request, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], shared_store: Annotated[SharedStore, Depends(get_shared_store)], app: Annotated[FastAPI, Depends(get_application)], ) -> TaskId: @@ -56,7 +59,7 @@ async def pull_user_servcices_docker_images( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=task_pull_user_servcices_docker_images, unique=True, app=app, @@ -85,7 +88,9 @@ async def pull_user_servcices_docker_images( async def create_service_containers_task( # pylint: disable=too-many-arguments request: Request, containers_create: ContainersCreate, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], settings: Annotated[ApplicationSettings, Depends(get_settings)], shared_store: Annotated[SharedStore, Depends(get_shared_store)], app: Annotated[FastAPI, Depends(get_application)], @@ -95,7 +100,7 @@ async def create_service_containers_task( # pylint: disable=too-many-arguments try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=task_create_service_containers, unique=True, settings=settings, @@ -117,7 +122,9 @@ async def create_service_containers_task( # pylint: disable=too-many-arguments @cancel_on_disconnect async def runs_docker_compose_down_task( request: Request, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], settings: Annotated[ApplicationSettings, Depends(get_settings)], shared_store: Annotated[SharedStore, Depends(get_shared_store)], app: Annotated[FastAPI, Depends(get_application)], @@ -127,7 +134,7 @@ async def runs_docker_compose_down_task( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=task_runs_docker_compose_down, unique=True, app=app, @@ -148,7 +155,9 @@ async def runs_docker_compose_down_task( @cancel_on_disconnect async def state_restore_task( request: Request, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], settings: Annotated[ApplicationSettings, Depends(get_settings)], mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], app: Annotated[FastAPI, Depends(get_application)], @@ -157,7 +166,7 @@ async def state_restore_task( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=task_restore_state, unique=True, settings=settings, @@ -177,7 +186,9 @@ async def state_restore_task( @cancel_on_disconnect async def state_save_task( request: Request, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], app: Annotated[FastAPI, Depends(get_application)], mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], settings: Annotated[ApplicationSettings, Depends(get_settings)], @@ -186,7 +197,7 @@ async def state_save_task( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=task_save_state, unique=True, settings=settings, @@ -206,7 +217,9 @@ async def state_save_task( @cancel_on_disconnect async def ports_inputs_pull_task( request: Request, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], app: Annotated[FastAPI, Depends(get_application)], settings: Annotated[ApplicationSettings, Depends(get_settings)], mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], @@ -217,7 +230,7 @@ async def ports_inputs_pull_task( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=task_ports_inputs_pull, unique=True, port_keys=port_keys, @@ -239,7 +252,9 @@ async def ports_inputs_pull_task( @cancel_on_disconnect async def ports_outputs_pull_task( request: Request, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], app: Annotated[FastAPI, Depends(get_application)], mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], port_keys: list[str] | None = None, @@ -248,7 +263,7 @@ async def ports_outputs_pull_task( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=task_ports_outputs_pull, unique=True, port_keys=port_keys, @@ -268,7 +283,9 @@ async def ports_outputs_pull_task( @cancel_on_disconnect async def ports_outputs_push_task( request: Request, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], outputs_manager: Annotated[OutputsManager, Depends(get_outputs_manager)], app: Annotated[FastAPI, Depends(get_application)], ) -> TaskId: @@ -276,7 +293,7 @@ async def ports_outputs_push_task( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=task_ports_outputs_push, unique=True, outputs_manager=outputs_manager, @@ -295,7 +312,9 @@ async def ports_outputs_push_task( @cancel_on_disconnect async def containers_restart_task( request: Request, - tasks_manager: Annotated[TasksManager, Depends(get_tasks_manager)], + long_running_manager: Annotated[ + FastAPILongRunningManager, Depends(get_long_running_manager) + ], app: Annotated[FastAPI, Depends(get_application)], settings: Annotated[ApplicationSettings, Depends(get_settings)], shared_store: Annotated[SharedStore, Depends(get_shared_store)], @@ -304,7 +323,7 @@ async def containers_restart_task( try: return start_task( - tasks_manager, + long_running_manager.tasks_manager, task=task_containers_restart, unique=True, app=app, From 37192b6a2b9382c7f0cf7df10bcf77cc2886891b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 12 Jun 2025 09:31:28 +0200 Subject: [PATCH 46/46] fixed test --- .../tests/unit/with_dbs/01/test_long_running_tasks.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_long_running_tasks.py b/services/web/server/tests/unit/with_dbs/01/test_long_running_tasks.py index c6f58f29ee1..0b5f601d8f0 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_long_running_tasks.py +++ b/services/web/server/tests/unit/with_dbs/01/test_long_running_tasks.py @@ -6,6 +6,7 @@ # pylint: disable=no-self-argument from typing import Any +from unittest.mock import Mock import pytest from aiohttp.test_utils import TestClient @@ -77,11 +78,14 @@ async def test_listing_tasks_with_list_inprocess_tasks_error( class _DummyTaskManager: def list_tasks(self, *args, **kwargs): - raise Exception() # pylint: disable=broad-exception-raised + raise Exception # pylint: disable=broad-exception-raised # noqa: TRY002 + + mock = Mock() + mock.tasks_manager = _DummyTaskManager() mocker.patch( - "servicelib.aiohttp.long_running_tasks._routes.get_tasks_manager", - return_value=_DummyTaskManager(), + "servicelib.aiohttp.long_running_tasks._routes.get_long_running_manager", + return_value=mock, ) _async_jobs_listing_path = client.app.router["get_async_jobs"].url_for()