From 556b259631a848b1b87ee11e4d2fe0c90078f25b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Oct 2024 13:48:51 +0200 Subject: [PATCH 01/12] update requirements --- .../dynamic-scheduler/requirements/_base.txt | 265 ++++++++++++------ .../dynamic-scheduler/requirements/_test.txt | 26 +- .../dynamic-scheduler/requirements/_tools.txt | 31 +- 3 files changed, 212 insertions(+), 110 deletions(-) diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 714f37a8b3e..8502347388d 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -1,30 +1,41 @@ -aio-pika==9.4.1 +aio-pika==9.4.3 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiocache==0.12.2 +aiocache==0.12.3 # via -r requirements/../../../packages/service-library/requirements/_base.in aiodebug==2.3.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.21.0 +aiodocker==0.23.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiofiles==23.2.1 +aiofiles==24.1.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiohttp==3.9.3 +aiohappyeyeballs==2.4.3 + # via aiohttp +aiohttp==3.10.10 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker -aiormq==6.8.0 +aiormq==6.8.1 # via aio-pika aiosignal==1.3.1 # via aiohttp -alembic==1.13.1 +alembic==1.13.3 # via -r requirements/../../../packages/postgres-database/requirements/_base.in -anyio==4.3.0 +annotated-types==0.7.0 + # via pydantic +anyio==4.6.2.post1 # via # fast-depends # faststream @@ -43,26 +54,33 @@ async-timeout==4.0.3 # via asyncpg asyncpg==0.29.0 # via sqlalchemy -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp # jsonschema # referencing bidict==0.23.1 # via python-socketio -certifi==2024.2.2 +certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore # httpx # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -74,25 +92,17 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator -email-validator==2.1.1 +email-validator==2.2.0 # via pydantic -fast-depends==2.4.2 +fast-depends==2.4.11 # via faststream -fastapi==0.99.1 +fastapi==0.115.2 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in - # prometheus-fastapi-instrumentator -faststream==0.5.10 +faststream==0.5.27 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via @@ -102,66 +112,80 @@ googleapis-common-protos==1.65.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -greenlet==3.0.3 +greenlet==3.1.1 # via sqlalchemy -grpcio==1.66.0 +grpcio==1.67.0 # via opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # httpcore # uvicorn # wsproto -httpcore==1.0.5 +httpcore==1.0.6 # via httpx -httptools==0.6.1 +httptools==0.6.2 # via uvicorn -httpx==0.27.0 +httpx==0.27.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in -idna==3.6 +idna==3.10 # via # anyio # email-validator # httpx # requests # yarl -importlib-metadata==8.0.0 +importlib-metadata==8.4.0 # via opentelemetry-api -jsonschema==4.21.1 +jsonschema==4.23.0 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in jsonschema-specifications==2023.7.1 # via jsonschema -mako==1.3.2 +mako==1.3.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.1 # via mako mdurl==0.1.2 # via markdown-it-py -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc @@ -172,125 +196,177 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests -orjson==3.10.0 +orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -packaging==24.0 +packaging==24.1 # via -r requirements/_base.in pamqp==3.3.0 # via aiormq -prometheus-client==0.20.0 +prometheus-client==0.21.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==6.1.0 +prometheus-fastapi-instrumentator==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==4.25.4 +propcache==0.2.0 + # via yarl +protobuf==4.25.5 # via # googleapis-common-protos # opentelemetry-proto psutil==6.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -psycopg2-binary==2.9.9 +psycopg2-binary==2.9.10 # via sqlalchemy -pydantic==1.10.15 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi -pygments==2.17.2 + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +pygments==2.18.0 # via rich -pyinstrument==4.6.2 +pyinstrument==5.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 # via arrow python-dotenv==1.0.1 - # via uvicorn -python-engineio==4.9.1 + # via + # pydantic-settings + # uvicorn +python-engineio==4.10.1 # via python-socketio -python-socketio==5.11.2 +python-socketio==5.11.4 # via -r requirements/_base.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in # uvicorn -redis==5.0.4 +redis==5.1.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -303,20 +379,20 @@ repro-zipfile==0.3.1 # via -r requirements/../../../packages/service-library/requirements/_base.in requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.7.1 +rich==13.9.2 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.18.0 +rpds-py==0.20.0 # via # jsonschema # referencing -setuptools==74.0.0 +setuptools==75.2.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer -simple-websocket==1.0.0 +simple-websocket==1.1.0 # via python-engineio six==1.16.0 # via python-dateutil @@ -324,70 +400,91 @@ sniffio==1.3.1 # via # anyio # httpx -sqlalchemy==1.4.52 +sqlalchemy==1.4.54 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic -starlette==0.27.0 +starlette==0.40.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi -tenacity==8.5.0 + # prometheus-fastapi-instrumentator +tenacity==9.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -tqdm==4.66.2 +tqdm==4.66.5 # via -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.12.3 +typer==0.12.5 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in - # faststream -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20241003 # via arrow -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via # aiodebug - # aiodocker # alembic # fastapi # faststream # opentelemetry-sdk # pydantic + # pydantic-core # typer -urllib3==2.2.2 +urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests -uvicorn==0.29.0 +uvicorn==0.32.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in -uvloop==0.19.0 +uvloop==0.21.0 # via uvicorn -watchfiles==0.21.0 +watchfiles==0.24.0 # via uvicorn -websockets==12.0 +websockets==13.1 # via uvicorn wrapt==1.16.0 # via @@ -395,11 +492,11 @@ wrapt==1.16.0 # opentelemetry-instrumentation wsproto==1.2.0 # via simple-websocket -yarl==1.9.4 +yarl==1.15.3 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in # aio-pika # aiohttp # aiormq -zipp==3.20.1 +zipp==3.20.2 # via importlib-metadata diff --git a/services/dynamic-scheduler/requirements/_test.txt b/services/dynamic-scheduler/requirements/_test.txt index b48cff66d52..44dbc740669 100644 --- a/services/dynamic-scheduler/requirements/_test.txt +++ b/services/dynamic-scheduler/requirements/_test.txt @@ -1,44 +1,44 @@ -anyio==4.3.0 +anyio==4.6.2.post1 # via # -c requirements/_base.txt # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -certifi==2024.2.2 +certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # httpcore # httpx # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -c requirements/_base.txt # requests -coverage==7.6.1 +coverage==7.6.3 # via # -r requirements/_test.in # pytest-cov docker==7.1.0 # via -r requirements/_test.in -faker==29.0.0 +faker==30.4.0 # via -r requirements/_test.in h11==0.14.0 # via # -c requirements/_base.txt # httpcore -httpcore==1.0.5 +httpcore==1.0.6 # via # -c requirements/_base.txt # httpx -httpx==0.27.0 +httpx==0.27.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # respx icdiff==2.0.7 # via pytest-icdiff -idna==3.6 +idna==3.10 # via # -c requirements/_base.txt # anyio @@ -46,7 +46,7 @@ idna==3.6 # requests iniconfig==2.0.0 # via pytest -packaging==24.0 +packaging==24.1 # via # -c requirements/_base.txt # pytest @@ -101,9 +101,13 @@ sniffio==1.3.1 # anyio # asgi-lifespan # httpx -termcolor==2.4.0 +termcolor==2.5.0 # via pytest-sugar -urllib3==2.2.2 +typing-extensions==4.12.2 + # via + # -c requirements/_base.txt + # faker +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/dynamic-scheduler/requirements/_tools.txt b/services/dynamic-scheduler/requirements/_tools.txt index 3f27c470fe3..d15ef99dd1f 100644 --- a/services/dynamic-scheduler/requirements/_tools.txt +++ b/services/dynamic-scheduler/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -13,9 +13,9 @@ click==8.1.7 # -c requirements/_base.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv @@ -27,7 +27,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.11.2 +mypy==1.12.0 # via -r requirements/../../../requirements/devenv.txt mypy-extensions==1.0.0 # via @@ -35,7 +35,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.1 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -52,32 +52,33 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.1 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # pre-commit -ruff==0.6.7 +ruff==0.6.9 # via -r requirements/../../../requirements/devenv.txt -setuptools==74.0.0 +setuptools==75.2.0 # via # -c requirements/_base.txt # pip-tools tomlkit==0.13.2 # via pylint -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via # -c requirements/_base.txt + # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.26.6 # via pre-commit wheel==0.44.0 # via pip-tools From 56ccda38d11a0de41d2827472e46e54e77e2274d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Oct 2024 14:37:23 +0200 Subject: [PATCH 02/12] ported dynamic-scheduler --- .../src/common_library/pydantic_validators.py | 20 ++++++++ .../dynamic_services.py | 7 ++- .../dynamic_services.py | 14 +++++- .../api_schemas_webserver/projects_nodes.py | 14 ++++-- .../src/settings_library/basic_types.py | 5 +- .../core/settings.py | 35 ++++++++----- .../models/schemas/meta.py | 22 ++++----- .../services/director_v2/_public_client.py | 19 ++++--- .../unit/api_rpc/test_api_rpc__services.py | 24 ++++++--- .../dynamic-scheduler/tests/unit/conftest.py | 14 ++++-- .../tests/unit/service_tracker/test__api.py | 49 +++++++++++++------ .../test_services_status_monitor__monitor.py | 32 ++++++++---- .../tests/unit/test__model_examples.py | 4 +- .../dynamic-scheduler/tests/unit/test_cli.py | 4 +- 14 files changed, 186 insertions(+), 77 deletions(-) create mode 100644 packages/common-library/src/common_library/pydantic_validators.py diff --git a/packages/common-library/src/common_library/pydantic_validators.py b/packages/common-library/src/common_library/pydantic_validators.py new file mode 100644 index 00000000000..40cb32f02a0 --- /dev/null +++ b/packages/common-library/src/common_library/pydantic_validators.py @@ -0,0 +1,20 @@ +import datetime + +from pydantic import field_validator + + +def _try_convert_str_to_float_or_return( + v: datetime.timedelta | str | float, +) -> datetime.timedelta | str | float: + if isinstance(v, str): + try: + return float(v) + except ValueError: + # returns format like "1:00:00" + return v + return v + + +def timedelta_try_convert_str_to_float(field: str): + """Transforms a float/int number into a valid datetime as it used to work in the past""" + return field_validator(field, mode="before")(_try_convert_str_to_float_or_return) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py index 151611271a4..092e2b0664e 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py @@ -1,6 +1,6 @@ -from typing import TypeAlias +from typing import Final, TypeAlias -from pydantic import BaseModel, ByteSize, ConfigDict, Field +from pydantic import BaseModel, ByteSize, ConfigDict, Field, TypeAdapter from ..resource_tracker import HardwareInfo, PricingInfo from ..services import ServicePortKey @@ -75,6 +75,9 @@ class DynamicServiceCreate(ServiceDetails): DynamicServiceGet: TypeAlias = RunningDynamicServiceDetails +DynamicServiceGetAdapter: Final[TypeAdapter[DynamicServiceGet]] = TypeAdapter( + DynamicServiceGet +) class GetProjectInactivityResponse(BaseModel): diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py index 47c4fc69a18..17eb85d2cd7 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py @@ -1,3 +1,5 @@ +from typing import Final + from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID @@ -5,7 +7,7 @@ from models_library.services_resources import ServiceResourcesDictHelpers from models_library.users import UserID from models_library.wallets import WalletInfo -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, TypeAdapter class DynamicServiceStart(DynamicServiceCreate): @@ -35,6 +37,11 @@ class DynamicServiceStart(DynamicServiceCreate): ) +DynamicServiceStartAdapter: Final[TypeAdapter[DynamicServiceStart]] = TypeAdapter( + DynamicServiceStart +) + + class DynamicServiceStop(BaseModel): user_id: UserID project_id: ProjectID @@ -53,3 +60,8 @@ class DynamicServiceStop(BaseModel): } } ) + + +DynamicServiceStopAdapter: Final[TypeAdapter[DynamicServiceStop]] = TypeAdapter( + DynamicServiceStop +) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 2e21f9d7c52..7422ded3fee 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -1,7 +1,7 @@ # mypy: disable-error-code=truthy-function -from typing import Any, Literal, TypeAlias +from typing import Any, Final, Literal, TypeAlias -from pydantic import ConfigDict, Field +from pydantic import ConfigDict, Field, TypeAdapter from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut from ..basic_types import PortInt @@ -106,7 +106,7 @@ class NodeGet(OutputSchema): "service_basepath": "/x/E1O2E-LAH", "service_state": "pending", "service_message": "no suitable node (insufficient resources on 1 node)", - "user_id": 123, + "user_id": "123", }, # dynamic { @@ -120,13 +120,16 @@ class NodeGet(OutputSchema): "service_basepath": "/x/E1O2E-LAH", "service_state": "pending", "service_message": "no suitable node (insufficient resources on 1 node)", - "user_id": 123, + "user_id": "123", }, ] } ) +NodeGetAdapter: Final[TypeAdapter[NodeGet]] = TypeAdapter(NodeGet) + + class NodeGetIdle(OutputSchema): service_state: Literal["idle"] service_uuid: NodeID @@ -145,6 +148,9 @@ def from_node_id(cls, node_id: NodeID) -> "NodeGetIdle": ) +NodeGetIdleAdapter: Final[TypeAdapter[NodeGetIdle]] = TypeAdapter(NodeGetIdle) + + class NodeGetUnknown(OutputSchema): service_state: Literal["unknown"] service_uuid: NodeID diff --git a/packages/settings-library/src/settings_library/basic_types.py b/packages/settings-library/src/settings_library/basic_types.py index 4c275397e1b..49fef50e9ef 100644 --- a/packages/settings-library/src/settings_library/basic_types.py +++ b/packages/settings-library/src/settings_library/basic_types.py @@ -4,9 +4,9 @@ # an extra dependency to a larger models_library (intra-repo library) from enum import Enum -from typing import Annotated, TypeAlias +from typing import Annotated, Final, TypeAlias -from pydantic import Field, StringConstraints +from pydantic import Field, StringConstraints, TypeAdapter # port number range PortInt: TypeAlias = Annotated[int, Field(gt=0, lt=65535)] @@ -14,6 +14,7 @@ # e.g. 'v5' VersionTag: TypeAlias = Annotated[str, StringConstraints(pattern=r"^v\d$")] +VersionTagAdapter: Final[TypeAdapter[VersionTag]] = TypeAdapter(VersionTag) class LogLevel(str, Enum): diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 9605619d57d..189f37f1b7c 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,9 +1,10 @@ import datetime from functools import cached_property -from pydantic import Field, parse_obj_as, validator +from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from pydantic import AliasChoices, ConfigDict, Field, field_validator from settings_library.application import BaseApplicationSettings -from settings_library.basic_types import LogLevel, VersionTag +from settings_library.basic_types import LogLevel, VersionTag, VersionTagAdapter from settings_library.director_v2 import DirectorV2Settings from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings @@ -19,20 +20,22 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): # CODE STATICS --------------------------------------------------------- API_VERSION: str = API_VERSION APP_NAME: str = PROJECT_NAME - API_VTAG: VersionTag = parse_obj_as(VersionTag, API_VTAG) + API_VTAG: VersionTag = VersionTagAdapter.validate_python(API_VTAG) # RUNTIME ----------------------------------------------------------- DYNAMIC_SCHEDULER__LOGLEVEL: LogLevel = Field( default=LogLevel.INFO, - env=["DYNAMIC_SCHEDULER__LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices( + "DYNAMIC_SCHEDULER__LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "DYNAMIC_SCHEDULER__LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) @@ -48,11 +51,17 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): def LOG_LEVEL(self): # noqa: N802 return self.DYNAMIC_SCHEDULER__LOGLEVEL - @validator("DYNAMIC_SCHEDULER__LOGLEVEL") + _try_convert_dynamic_scheduler_stop_service_timeout = ( + timedelta_try_convert_str_to_float("DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT") + ) + + @field_validator("DYNAMIC_SCHEDULER__LOGLEVEL") @classmethod def valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) + model_config = ConfigDict(extra="allow") + class ApplicationSettings(_BaseApplicationSettings): """Web app's environment variables @@ -61,11 +70,13 @@ class ApplicationSettings(_BaseApplicationSettings): """ DYNAMIC_SCHEDULER_RABBITMQ: RabbitSettings = Field( - auto_default_from_env=True, description="settings for service/rabbitmq" + json_schema_extra={"auto_default_from_env": True}, + description="settings for service/rabbitmq", ) DYNAMIC_SCHEDULER_REDIS: RedisSettings = Field( - auto_default_from_env=True, description="settings for service/redis" + json_schema_extra={"auto_default_from_env": True}, + description="settings for service/redis", ) DYNAMIC_SCHEDULER_SWAGGER_API_DOC_ENABLED: bool = Field( @@ -73,12 +84,14 @@ class ApplicationSettings(_BaseApplicationSettings): ) DYNAMIC_SCHEDULER_DIRECTOR_V2_SETTINGS: DirectorV2Settings = Field( - auto_default_from_env=True, description="settings for director-v2 service" + json_schema_extra={"auto_default_from_env": True}, + description="settings for director-v2 service", ) DYNAMIC_SCHEDULER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True DYNAMIC_SCHEDULER_PROFILING: bool = False DYNAMIC_SCHEDULER_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + json_schema_extra={"auto_default_from_env": True}, + description="settings for opentelemetry tracing", ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/models/schemas/meta.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/models/schemas/meta.py index df9d3fa4315..ad73c58ac70 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/models/schemas/meta.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/models/schemas/meta.py @@ -1,17 +1,17 @@ -from typing import Any, ClassVar - from models_library.api_schemas__common.meta import BaseMeta -from pydantic import HttpUrl +from pydantic import ConfigDict, HttpUrl class Meta(BaseMeta): docs_url: HttpUrl - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { - "example": { - "name": "simcore_service_dynamic_scheduler", - "version": "2.4.45", - "docs_url": "https://foo.io/doc", - } + model_config = ConfigDict( + json_schema_extra={ + "examples": [ + { + "name": "simcore_service_dynamic_scheduler", + "version": "2.4.45", + "docs_url": "https://foo.io/doc", + } + ] } + ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py index fd5ce9a2cb2..10996dbba0b 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py @@ -2,11 +2,18 @@ from typing import Any from fastapi import FastAPI, status -from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet +from models_library.api_schemas_directorv2.dynamic_services import ( + DynamicServiceGet, + DynamicServiceGetAdapter, +) from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, ) -from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle +from models_library.api_schemas_webserver.projects_nodes import ( + NodeGet, + NodeGetAdapter, + NodeGetIdle, +) from models_library.projects_nodes_io import NodeID from servicelib.fastapi.app_state import SingletonInAppStateMixin from servicelib.fastapi.http_client import AttachLifespanMixin, HasClientSetupInterface @@ -43,9 +50,9 @@ async def get_status( # in case of legacy version # we need to transfer the correct format! if "data" in dict_response: - return NodeGet.parse_obj(dict_response["data"]) + return NodeGetAdapter.validate_python(dict_response["data"]) - return DynamicServiceGet.parse_obj(dict_response) + return DynamicServiceGetAdapter.validate_python(dict_response) except UnexpectedStatusError as e: if ( e.response.status_code # type: ignore[attr-defined] # pylint:disable=no-member @@ -62,9 +69,9 @@ async def run_dynamic_service( # legacy services if "data" in dict_response: - return NodeGet.parse_obj(dict_response["data"]) + return NodeGetAdapter.validate_python(dict_response["data"]) - return DynamicServiceGet.parse_obj(dict_response) + return DynamicServiceGetAdapter.validate_python(dict_response) async def stop_dynamic_service( self, diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py index c484f722ff9..2defd58f163 100644 --- a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py +++ b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py @@ -9,12 +9,20 @@ from faker import Faker from fastapi import FastAPI, status from fastapi.encoders import jsonable_encoder -from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet +from models_library.api_schemas_directorv2.dynamic_services import ( + DynamicServiceGet, + DynamicServiceGetAdapter, +) from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, + DynamicServiceStartAdapter, DynamicServiceStop, ) -from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle +from models_library.api_schemas_webserver.projects_nodes import ( + NodeGet, + NodeGetAdapter, + NodeGetIdle, +) from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID @@ -52,14 +60,16 @@ def node_not_found(faker: Faker) -> NodeID: @pytest.fixture def service_status_new_style() -> DynamicServiceGet: - return DynamicServiceGet.parse_obj( - DynamicServiceGet.Config.schema_extra["examples"][1] + return DynamicServiceGetAdapter.validate_python( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] ) @pytest.fixture def service_status_legacy() -> NodeGet: - return NodeGet.parse_obj(NodeGet.Config.schema_extra["examples"][1]) + return NodeGetAdapter.validate_python( + NodeGet.model_config["json_schema_extra"]["examples"][1] + ) @pytest.fixture @@ -173,8 +183,8 @@ async def test_get_state( @pytest.fixture def dynamic_service_start() -> DynamicServiceStart: # one for legacy and one for new style? - return DynamicServiceStart.parse_obj( - DynamicServiceStart.Config.schema_extra["example"] + return DynamicServiceStartAdapter.validate_python( + DynamicServiceStart.model_config["json_schema_extra"]["example"] ) diff --git a/services/dynamic-scheduler/tests/unit/conftest.py b/services/dynamic-scheduler/tests/unit/conftest.py index 642ed2170ce..43e7209d297 100644 --- a/services/dynamic-scheduler/tests/unit/conftest.py +++ b/services/dynamic-scheduler/tests/unit/conftest.py @@ -4,7 +4,9 @@ import pytest from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, + DynamicServiceStartAdapter, DynamicServiceStop, + DynamicServiceStopAdapter, ) from models_library.projects_nodes_io import NodeID @@ -12,9 +14,11 @@ @pytest.fixture def get_dynamic_service_start() -> Callable[[NodeID], DynamicServiceStart]: def _(node_id: NodeID) -> DynamicServiceStart: - dict_data = deepcopy(DynamicServiceStart.Config.schema_extra["example"]) + dict_data = deepcopy( + DynamicServiceStart.model_config["json_schema_extra"]["example"] + ) dict_data["service_uuid"] = f"{node_id}" - return DynamicServiceStart.parse_obj(dict_data) + return DynamicServiceStartAdapter.validate_python(dict_data) return _ @@ -22,8 +26,10 @@ def _(node_id: NodeID) -> DynamicServiceStart: @pytest.fixture def get_dynamic_service_stop() -> Callable[[NodeID], DynamicServiceStop]: def _(node_id: NodeID) -> DynamicServiceStop: - dict_data = deepcopy(DynamicServiceStop.Config.schema_extra["example"]) + dict_data = deepcopy( + DynamicServiceStop.model_config["json_schema_extra"]["example"] + ) dict_data["node_id"] = f"{node_id}" - return DynamicServiceStop.parse_obj(dict_data) + return DynamicServiceStopAdapter.validate_python(dict_data) return _ diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py index 0755f7e5d78..b1d31b18761 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py @@ -9,12 +9,20 @@ import pytest from faker import Faker from fastapi import FastAPI -from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet +from models_library.api_schemas_directorv2.dynamic_services import ( + DynamicServiceGet, + DynamicServiceGetAdapter, +) from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, DynamicServiceStop, ) -from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle +from models_library.api_schemas_webserver.projects_nodes import ( + NodeGet, + NodeGetAdapter, + NodeGetIdle, + NodeGetIdleAdapter, +) from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceState from pydantic import NonNegativeInt @@ -115,12 +123,17 @@ async def test_services_tracer_workflow( @pytest.mark.parametrize( "status", [ - *[NodeGet.parse_obj(o) for o in NodeGet.Config.schema_extra["examples"]], *[ - DynamicServiceGet.parse_obj(o) - for o in DynamicServiceGet.Config.schema_extra["examples"] + NodeGet.model_validate(o) + for o in NodeGet.model_config["json_schema_extra"]["examples"] + ], + *[ + DynamicServiceGet.model_validate(o) + for o in DynamicServiceGet.model_config["json_schema_extra"]["examples"] ], - NodeGetIdle.parse_obj(NodeGetIdle.Config.schema_extra["example"]), + NodeGetIdle.model_validate( + NodeGetIdle.model_config["json_schema_extra"]["example"] + ), ], ) async def test_set_if_status_changed( @@ -162,15 +175,19 @@ async def test_set_service_status_task_uid( "status, expected_poll_interval", [ ( - NodeGet.parse_obj(NodeGet.Config.schema_extra["examples"][1]), + NodeGetAdapter.validate_python( + NodeGet.model_config["json_schema_extra"]["examples"][1] + ), _LOW_RATE_POLL_INTERVAL, ), *[ - (DynamicServiceGet.parse_obj(o), NORMAL_RATE_POLL_INTERVAL) - for o in DynamicServiceGet.Config.schema_extra["examples"] + (DynamicServiceGetAdapter.validate_python(o), NORMAL_RATE_POLL_INTERVAL) + for o in DynamicServiceGet.model_config["json_schema_extra"]["examples"] ], ( - NodeGetIdle.parse_obj(NodeGetIdle.Config.schema_extra["example"]), + NodeGetIdleAdapter.validate_python( + NodeGetIdle.model_config["json_schema_extra"]["example"] + ), _LOW_RATE_POLL_INTERVAL, ), ], @@ -182,23 +199,25 @@ def test__get_poll_interval( def _get_node_get_from(service_state: ServiceState) -> NodeGet: - dict_data = NodeGet.Config.schema_extra["examples"][1] + dict_data = NodeGet.model_config["json_schema_extra"]["examples"][1] assert "service_state" in dict_data dict_data["service_state"] = service_state - return NodeGet.parse_obj(dict_data) + return NodeGetAdapter.validate_python(dict_data) def _get_dynamic_service_get_from( service_state: ServiceState, ) -> DynamicServiceGet: - dict_data = DynamicServiceGet.Config.schema_extra["examples"][1] + dict_data = DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] assert "state" in dict_data dict_data["state"] = service_state - return DynamicServiceGet.parse_obj(dict_data) + return DynamicServiceGetAdapter.validate_python(dict_data) def _get_node_get_idle() -> NodeGetIdle: - return NodeGetIdle.parse_obj(NodeGetIdle.Config.schema_extra["example"]) + return NodeGetIdleAdapter.validate_python( + NodeGetIdle.model_config["json_schema_extra"]["example"] + ) def __get_flat_list(nested_list: list[list[Any]]) -> list[Any]: diff --git a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py index 2dd5270b627..536cf524785 100644 --- a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py +++ b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py @@ -15,12 +15,20 @@ from fastapi import FastAPI, status from fastapi.encoders import jsonable_encoder from httpx import Request, Response -from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet +from models_library.api_schemas_directorv2.dynamic_services import ( + DynamicServiceGet, + DynamicServiceGetAdapter, +) from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, DynamicServiceStop, ) -from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle +from models_library.api_schemas_webserver.projects_nodes import ( + NodeGet, + NodeGetAdapter, + NodeGetIdle, + NodeGetIdleAdapter, +) from models_library.projects_nodes_io import NodeID from pydantic import NonNegativeInt from pytest_mock import MockerFixture @@ -69,7 +77,7 @@ def _add_to_dict(dict_data: dict, entries: list[tuple[str, Any]]) -> None: def _get_node_get_with(state: str, node_id: NodeID = _DEFAULT_NODE_ID) -> NodeGet: - dict_data = deepcopy(NodeGet.Config.schema_extra["examples"][1]) + dict_data = deepcopy(NodeGet.model_config["json_schema_extra"]["examples"][1]) _add_to_dict( dict_data, [ @@ -77,13 +85,15 @@ def _get_node_get_with(state: str, node_id: NodeID = _DEFAULT_NODE_ID) -> NodeGe ("service_uuid", f"{node_id}"), ], ) - return NodeGet.parse_obj(dict_data) + return NodeGetAdapter.validate_python(dict_data) def _get_dynamic_service_get_legacy_with( state: str, node_id: NodeID = _DEFAULT_NODE_ID ) -> DynamicServiceGet: - dict_data = deepcopy(DynamicServiceGet.Config.schema_extra["examples"][0]) + dict_data = deepcopy( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][0] + ) _add_to_dict( dict_data, [ @@ -92,13 +102,15 @@ def _get_dynamic_service_get_legacy_with( ("node_uuid", f"{node_id}"), ], ) - return DynamicServiceGet.parse_obj(dict_data) + return DynamicServiceGetAdapter.validate_python(dict_data) def _get_dynamic_service_get_new_style_with( state: str, node_id: NodeID = _DEFAULT_NODE_ID ) -> DynamicServiceGet: - dict_data = deepcopy(DynamicServiceGet.Config.schema_extra["examples"][1]) + dict_data = deepcopy( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] + ) _add_to_dict( dict_data, [ @@ -107,18 +119,18 @@ def _get_dynamic_service_get_new_style_with( ("node_uuid", f"{node_id}"), ], ) - return DynamicServiceGet.parse_obj(dict_data) + return DynamicServiceGetAdapter.validate_python(dict_data) def _get_node_get_idle(node_id: NodeID = _DEFAULT_NODE_ID) -> NodeGetIdle: - dict_data = NodeGetIdle.Config.schema_extra["example"] + dict_data = NodeGetIdle.model_config["json_schema_extra"]["example"] _add_to_dict( dict_data, [ ("service_uuid", f"{node_id}"), ], ) - return NodeGetIdle.parse_obj(dict_data) + return NodeGetIdleAdapter.validate_python(dict_data) class _ResponseTimeline: diff --git a/services/dynamic-scheduler/tests/unit/test__model_examples.py b/services/dynamic-scheduler/tests/unit/test__model_examples.py index 858bcc66a4d..e768927cfe4 100644 --- a/services/dynamic-scheduler/tests/unit/test__model_examples.py +++ b/services/dynamic-scheduler/tests/unit/test__model_examples.py @@ -3,7 +3,7 @@ import pytest import simcore_service_dynamic_scheduler.models -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from pytest_simcore.pydantic_models import walk_model_examples_in_package @@ -15,7 +15,7 @@ def test_api_server_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): try: - assert model_cls.parse_obj(example_data) is not None + assert TypeAdapter(model_cls).validate_python(example_data) is not None except ValidationError as err: pytest.fail( f"\n{example_name}: {json.dumps(example_data, indent=1)}\nError: {err}" diff --git a/services/dynamic-scheduler/tests/unit/test_cli.py b/services/dynamic-scheduler/tests/unit/test_cli.py index 2e812f7e118..85b2a5e2dcd 100644 --- a/services/dynamic-scheduler/tests/unit/test_cli.py +++ b/services/dynamic-scheduler/tests/unit/test_cli.py @@ -39,8 +39,8 @@ def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): assert result.exit_code == os.EX_OK, result.output print(result.output) - settings = ApplicationSettings.parse_raw(result.output) - assert settings == ApplicationSettings.create_from_envs() + settings = ApplicationSettings.model_validate_json(result.output) + assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() def test_main(app_environment: EnvVarsDict): From 143f13cfb82ecc48f793f348935d5ba883e2aa8c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Oct 2024 14:43:49 +0200 Subject: [PATCH 03/12] using not deprecated forms --- .../services/service_tracker/_api.py | 2 +- .../tests/unit/api_rest/test_api_rest__meta.py | 2 +- .../tests/unit/api_rpc/test_api_rpc__services.py | 12 ++++++++---- .../tests/unit/service_tracker/test__api.py | 2 +- .../test_services_status_monitor__monitor.py | 6 ++++-- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py index 1b1b4a0d9f8..99215c69123 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py @@ -150,7 +150,7 @@ async def set_if_status_changed_for_service( model.scheduled_to_run = False # check if model changed - json_status = status.json() + json_status = status.model_dump_json() if model.service_status != json_status: model.service_status = json_status model.current_state = _get_current_scheduler_service_state( diff --git a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py index 8d986dfe60e..ccf9aeab911 100644 --- a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py +++ b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py @@ -9,4 +9,4 @@ async def test_health(client: AsyncClient): response = await client.get(f"/{API_VTAG}/meta") assert response.status_code == status.HTTP_200_OK - assert Meta.parse_raw(response.text) + assert Meta.model_validate_json(response.text) diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py index 2defd58f163..5c627015d7a 100644 --- a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py +++ b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py @@ -91,7 +91,9 @@ def mock_director_v0_service_state( ) as mock: mock.get(f"/fake-status/{node_id_legacy}").respond( status.HTTP_200_OK, - text=json.dumps(jsonable_encoder({"data": service_status_legacy.dict()})), + text=json.dumps( + jsonable_encoder({"data": service_status_legacy.model_dump()}) + ), ) # service was not found response @@ -114,7 +116,7 @@ def mock_director_v2_service_state( assert_all_mocked=True, # IMPORTANT: KEEP always True! ) as mock: mock.get(f"/dynamic_services/{node_id_new_style}").respond( - status.HTTP_200_OK, text=service_status_new_style.json() + status.HTTP_200_OK, text=service_status_new_style.model_dump_json() ) # emulate redirect response to director-v0 @@ -199,7 +201,9 @@ def mock_director_v0_service_run( ) as mock: mock.post("/fake-service-run").respond( status.HTTP_201_CREATED, - text=json.dumps(jsonable_encoder({"data": service_status_legacy.dict()})), + text=json.dumps( + jsonable_encoder({"data": service_status_legacy.model_dump()}) + ), ) yield None @@ -226,7 +230,7 @@ def mock_director_v2_service_run( else: request.respond( status.HTTP_201_CREATED, - text=service_status_new_style.json(), + text=service_status_new_style.model_dump_json(), ) yield None diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py index b1d31b18761..91594f725a6 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py @@ -151,7 +151,7 @@ async def test_set_if_status_changed( model = await get_tracked_service(app, node_id) assert model - assert model.service_status == status.json() + assert model.service_status == status.model_dump_json() async def test_set_service_status_task_uid( diff --git a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py index 536cf524785..e4b37cee4f8 100644 --- a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py +++ b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py @@ -221,10 +221,12 @@ def _side_effect_node_status_response(request: Request) -> Response: if isinstance(service_status, NodeGet): return Response( status.HTTP_200_OK, - text=json.dumps(jsonable_encoder({"data": service_status.dict()})), + text=json.dumps( + jsonable_encoder({"data": service_status.model_dump()}) + ), ) if isinstance(service_status, DynamicServiceGet): - return Response(status.HTTP_200_OK, text=service_status.json()) + return Response(status.HTTP_200_OK, text=service_status.model_dump_json()) if isinstance(service_status, NodeGetIdle): return Response(status.HTTP_404_NOT_FOUND) From f9dbe53c04e5e4ce58d349b8b2f14cad13ecdd32 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Oct 2024 14:51:58 +0200 Subject: [PATCH 04/12] mypy --- .../simcore_service_dynamic_scheduler/api/rest/routes.py | 2 +- .../src/simcore_service_dynamic_scheduler/core/errors.py | 6 +++--- .../src/simcore_service_dynamic_scheduler/core/settings.py | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/routes.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/routes.py index 8c1d3e21ed8..2b938584780 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/routes.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/routes.py @@ -16,4 +16,4 @@ def setup_rest_api(app: FastAPI): app.include_router(api_router) app.add_exception_handler(Exception, handle_errors_as_500) - app.add_exception_handler(HTTPException, http_exception_as_json_response) + app.add_exception_handler(HTTPException, http_exception_as_json_response) # type: ignore[arg-type] diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py index 260202d00f4..2677b7bc370 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py @@ -1,5 +1,5 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class BaseDynamicSchedulerError(PydanticErrorMixin, ValueError): - code = "simcore.service.dynamic.scheduler" +class BaseDynamicSchedulerError(OsparcErrorMixin, ValueError): + code = "simcore.service.dynamic.scheduler" # type:ignore[assignment] diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 189f37f1b7c..41dc283a6b1 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -2,7 +2,8 @@ from functools import cached_property from common_library.pydantic_validators import timedelta_try_convert_str_to_float -from pydantic import AliasChoices, ConfigDict, Field, field_validator +from pydantic import AliasChoices, Field, field_validator +from pydantic_settings import SettingsConfigDict from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag, VersionTagAdapter from settings_library.director_v2 import DirectorV2Settings @@ -60,7 +61,7 @@ def LOG_LEVEL(self): # noqa: N802 def valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) - model_config = ConfigDict(extra="allow") + model_config = SettingsConfigDict(extra="allow") class ApplicationSettings(_BaseApplicationSettings): From c28d1ccf4a6bc46a2ef2b293bf83410a01e0df9d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 18 Oct 2024 08:35:32 +0200 Subject: [PATCH 05/12] removed adapter pattern --- .../api_schemas_directorv2/dynamic_services.py | 7 ++----- .../services/director_v2/_public_client.py | 10 ++++------ .../tests/unit/api_rpc/test_api_rpc__services.py | 8 +++----- .../tests/unit/service_tracker/test__api.py | 14 +++++++------- .../test_services_status_monitor__monitor.py | 11 ++++------- 5 files changed, 20 insertions(+), 30 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py index 092e2b0664e..151611271a4 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py @@ -1,6 +1,6 @@ -from typing import Final, TypeAlias +from typing import TypeAlias -from pydantic import BaseModel, ByteSize, ConfigDict, Field, TypeAdapter +from pydantic import BaseModel, ByteSize, ConfigDict, Field from ..resource_tracker import HardwareInfo, PricingInfo from ..services import ServicePortKey @@ -75,9 +75,6 @@ class DynamicServiceCreate(ServiceDetails): DynamicServiceGet: TypeAlias = RunningDynamicServiceDetails -DynamicServiceGetAdapter: Final[TypeAdapter[DynamicServiceGet]] = TypeAdapter( - DynamicServiceGet -) class GetProjectInactivityResponse(BaseModel): diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py index 10996dbba0b..b1edda1fd11 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py @@ -2,10 +2,7 @@ from typing import Any from fastapi import FastAPI, status -from models_library.api_schemas_directorv2.dynamic_services import ( - DynamicServiceGet, - DynamicServiceGetAdapter, -) +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, ) @@ -15,6 +12,7 @@ NodeGetIdle, ) from models_library.projects_nodes_io import NodeID +from pydantic import TypeAdapter from servicelib.fastapi.app_state import SingletonInAppStateMixin from servicelib.fastapi.http_client import AttachLifespanMixin, HasClientSetupInterface from servicelib.fastapi.http_client_thin import UnexpectedStatusError @@ -52,7 +50,7 @@ async def get_status( if "data" in dict_response: return NodeGetAdapter.validate_python(dict_response["data"]) - return DynamicServiceGetAdapter.validate_python(dict_response) + return TypeAdapter(DynamicServiceGet).validate_python(dict_response) except UnexpectedStatusError as e: if ( e.response.status_code # type: ignore[attr-defined] # pylint:disable=no-member @@ -71,7 +69,7 @@ async def run_dynamic_service( if "data" in dict_response: return NodeGetAdapter.validate_python(dict_response["data"]) - return DynamicServiceGetAdapter.validate_python(dict_response) + return TypeAdapter(DynamicServiceGet).validate_python(dict_response) async def stop_dynamic_service( self, diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py index 5c627015d7a..8eeba451cc8 100644 --- a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py +++ b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py @@ -9,10 +9,7 @@ from faker import Faker from fastapi import FastAPI, status from fastapi.encoders import jsonable_encoder -from models_library.api_schemas_directorv2.dynamic_services import ( - DynamicServiceGet, - DynamicServiceGetAdapter, -) +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, DynamicServiceStartAdapter, @@ -26,6 +23,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient, RPCServerError @@ -60,7 +58,7 @@ def node_not_found(faker: Faker) -> NodeID: @pytest.fixture def service_status_new_style() -> DynamicServiceGet: - return DynamicServiceGetAdapter.validate_python( + return TypeAdapter(DynamicServiceGet).validate_python( DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] ) diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py index 91594f725a6..7089675aad7 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py @@ -9,10 +9,7 @@ import pytest from faker import Faker from fastapi import FastAPI -from models_library.api_schemas_directorv2.dynamic_services import ( - DynamicServiceGet, - DynamicServiceGetAdapter, -) +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, DynamicServiceStop, @@ -25,7 +22,7 @@ ) from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceState -from pydantic import NonNegativeInt +from pydantic import NonNegativeInt, TypeAdapter from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.deferred_tasks import TaskUID from servicelib.utils import limited_gather @@ -181,7 +178,10 @@ async def test_set_service_status_task_uid( _LOW_RATE_POLL_INTERVAL, ), *[ - (DynamicServiceGetAdapter.validate_python(o), NORMAL_RATE_POLL_INTERVAL) + ( + TypeAdapter(DynamicServiceGet).validate_python(o), + NORMAL_RATE_POLL_INTERVAL, + ) for o in DynamicServiceGet.model_config["json_schema_extra"]["examples"] ], ( @@ -211,7 +211,7 @@ def _get_dynamic_service_get_from( dict_data = DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] assert "state" in dict_data dict_data["state"] = service_state - return DynamicServiceGetAdapter.validate_python(dict_data) + return TypeAdapter(DynamicServiceGet).validate_python(dict_data) def _get_node_get_idle() -> NodeGetIdle: diff --git a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py index e4b37cee4f8..9a60347a589 100644 --- a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py +++ b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py @@ -15,10 +15,7 @@ from fastapi import FastAPI, status from fastapi.encoders import jsonable_encoder from httpx import Request, Response -from models_library.api_schemas_directorv2.dynamic_services import ( - DynamicServiceGet, - DynamicServiceGetAdapter, -) +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, DynamicServiceStop, @@ -30,7 +27,7 @@ NodeGetIdleAdapter, ) from models_library.projects_nodes_io import NodeID -from pydantic import NonNegativeInt +from pydantic import NonNegativeInt, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.rabbit import RabbitSettings @@ -102,7 +99,7 @@ def _get_dynamic_service_get_legacy_with( ("node_uuid", f"{node_id}"), ], ) - return DynamicServiceGetAdapter.validate_python(dict_data) + return TypeAdapter(DynamicServiceGet).validate_python(dict_data) def _get_dynamic_service_get_new_style_with( @@ -119,7 +116,7 @@ def _get_dynamic_service_get_new_style_with( ("node_uuid", f"{node_id}"), ], ) - return DynamicServiceGetAdapter.validate_python(dict_data) + return TypeAdapter(DynamicServiceGet).validate_python(dict_data) def _get_node_get_idle(node_id: NodeID = _DEFAULT_NODE_ID) -> NodeGetIdle: From 79dc0e150ccbc28bd6f02b8068534d0c3bfcaba6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 18 Oct 2024 08:37:05 +0200 Subject: [PATCH 06/12] revert adapter pattern --- .../dynamic_services.py | 14 +------------- .../tests/unit/api_rpc/test_api_rpc__services.py | 3 +-- services/dynamic-scheduler/tests/unit/conftest.py | 7 +++---- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py index 17eb85d2cd7..47c4fc69a18 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py @@ -1,5 +1,3 @@ -from typing import Final - from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID @@ -7,7 +5,7 @@ from models_library.services_resources import ServiceResourcesDictHelpers from models_library.users import UserID from models_library.wallets import WalletInfo -from pydantic import BaseModel, ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict class DynamicServiceStart(DynamicServiceCreate): @@ -37,11 +35,6 @@ class DynamicServiceStart(DynamicServiceCreate): ) -DynamicServiceStartAdapter: Final[TypeAdapter[DynamicServiceStart]] = TypeAdapter( - DynamicServiceStart -) - - class DynamicServiceStop(BaseModel): user_id: UserID project_id: ProjectID @@ -60,8 +53,3 @@ class DynamicServiceStop(BaseModel): } } ) - - -DynamicServiceStopAdapter: Final[TypeAdapter[DynamicServiceStop]] = TypeAdapter( - DynamicServiceStop -) diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py index 8eeba451cc8..9ef7d96cbcb 100644 --- a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py +++ b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py @@ -12,7 +12,6 @@ from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, - DynamicServiceStartAdapter, DynamicServiceStop, ) from models_library.api_schemas_webserver.projects_nodes import ( @@ -183,7 +182,7 @@ async def test_get_state( @pytest.fixture def dynamic_service_start() -> DynamicServiceStart: # one for legacy and one for new style? - return DynamicServiceStartAdapter.validate_python( + return TypeAdapter(DynamicServiceStart).validate_python( DynamicServiceStart.model_config["json_schema_extra"]["example"] ) diff --git a/services/dynamic-scheduler/tests/unit/conftest.py b/services/dynamic-scheduler/tests/unit/conftest.py index 43e7209d297..a25596bd4f2 100644 --- a/services/dynamic-scheduler/tests/unit/conftest.py +++ b/services/dynamic-scheduler/tests/unit/conftest.py @@ -4,11 +4,10 @@ import pytest from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, - DynamicServiceStartAdapter, DynamicServiceStop, - DynamicServiceStopAdapter, ) from models_library.projects_nodes_io import NodeID +from pydantic import TypeAdapter @pytest.fixture @@ -18,7 +17,7 @@ def _(node_id: NodeID) -> DynamicServiceStart: DynamicServiceStart.model_config["json_schema_extra"]["example"] ) dict_data["service_uuid"] = f"{node_id}" - return DynamicServiceStartAdapter.validate_python(dict_data) + return TypeAdapter(DynamicServiceStart).validate_python(dict_data) return _ @@ -30,6 +29,6 @@ def _(node_id: NodeID) -> DynamicServiceStop: DynamicServiceStop.model_config["json_schema_extra"]["example"] ) dict_data["node_id"] = f"{node_id}" - return DynamicServiceStopAdapter.validate_python(dict_data) + return TypeAdapter(DynamicServiceStop).validate_python(dict_data) return _ From 237eee5535082450159cee370c1cf487f28c3f16 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 18 Oct 2024 08:38:53 +0200 Subject: [PATCH 07/12] revert adapter pattern --- .../api_schemas_webserver/projects_nodes.py | 10 ++-------- .../services/director_v2/_public_client.py | 10 +++------- .../tests/unit/api_rpc/test_api_rpc__services.py | 8 ++------ .../tests/unit/service_tracker/test__api.py | 15 +++++---------- .../test_services_status_monitor__monitor.py | 11 +++-------- 5 files changed, 15 insertions(+), 39 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 7422ded3fee..649d23f12ce 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -1,7 +1,7 @@ # mypy: disable-error-code=truthy-function -from typing import Any, Final, Literal, TypeAlias +from typing import Any, Literal, TypeAlias -from pydantic import ConfigDict, Field, TypeAdapter +from pydantic import ConfigDict, Field from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut from ..basic_types import PortInt @@ -127,9 +127,6 @@ class NodeGet(OutputSchema): ) -NodeGetAdapter: Final[TypeAdapter[NodeGet]] = TypeAdapter(NodeGet) - - class NodeGetIdle(OutputSchema): service_state: Literal["idle"] service_uuid: NodeID @@ -148,9 +145,6 @@ def from_node_id(cls, node_id: NodeID) -> "NodeGetIdle": ) -NodeGetIdleAdapter: Final[TypeAdapter[NodeGetIdle]] = TypeAdapter(NodeGetIdle) - - class NodeGetUnknown(OutputSchema): service_state: Literal["unknown"] service_uuid: NodeID diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py index b1edda1fd11..5ee4ae3bcac 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py @@ -6,11 +6,7 @@ from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStart, ) -from models_library.api_schemas_webserver.projects_nodes import ( - NodeGet, - NodeGetAdapter, - NodeGetIdle, -) +from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle from models_library.projects_nodes_io import NodeID from pydantic import TypeAdapter from servicelib.fastapi.app_state import SingletonInAppStateMixin @@ -48,7 +44,7 @@ async def get_status( # in case of legacy version # we need to transfer the correct format! if "data" in dict_response: - return NodeGetAdapter.validate_python(dict_response["data"]) + return TypeAdapter(NodeGet).validate_python(dict_response["data"]) return TypeAdapter(DynamicServiceGet).validate_python(dict_response) except UnexpectedStatusError as e: @@ -67,7 +63,7 @@ async def run_dynamic_service( # legacy services if "data" in dict_response: - return NodeGetAdapter.validate_python(dict_response["data"]) + return TypeAdapter(NodeGet).validate_python(dict_response["data"]) return TypeAdapter(DynamicServiceGet).validate_python(dict_response) diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py index 9ef7d96cbcb..7ee876e9e4b 100644 --- a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py +++ b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py @@ -14,11 +14,7 @@ DynamicServiceStart, DynamicServiceStop, ) -from models_library.api_schemas_webserver.projects_nodes import ( - NodeGet, - NodeGetAdapter, - NodeGetIdle, -) +from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID @@ -64,7 +60,7 @@ def service_status_new_style() -> DynamicServiceGet: @pytest.fixture def service_status_legacy() -> NodeGet: - return NodeGetAdapter.validate_python( + return TypeAdapter(NodeGet).validate_python( NodeGet.model_config["json_schema_extra"]["examples"][1] ) diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py index 7089675aad7..489dd14f2b8 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py @@ -14,12 +14,7 @@ DynamicServiceStart, DynamicServiceStop, ) -from models_library.api_schemas_webserver.projects_nodes import ( - NodeGet, - NodeGetAdapter, - NodeGetIdle, - NodeGetIdleAdapter, -) +from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceState from pydantic import NonNegativeInt, TypeAdapter @@ -172,7 +167,7 @@ async def test_set_service_status_task_uid( "status, expected_poll_interval", [ ( - NodeGetAdapter.validate_python( + TypeAdapter(NodeGet).validate_python( NodeGet.model_config["json_schema_extra"]["examples"][1] ), _LOW_RATE_POLL_INTERVAL, @@ -185,7 +180,7 @@ async def test_set_service_status_task_uid( for o in DynamicServiceGet.model_config["json_schema_extra"]["examples"] ], ( - NodeGetIdleAdapter.validate_python( + TypeAdapter(NodeGetIdle).validate_python( NodeGetIdle.model_config["json_schema_extra"]["example"] ), _LOW_RATE_POLL_INTERVAL, @@ -202,7 +197,7 @@ def _get_node_get_from(service_state: ServiceState) -> NodeGet: dict_data = NodeGet.model_config["json_schema_extra"]["examples"][1] assert "service_state" in dict_data dict_data["service_state"] = service_state - return NodeGetAdapter.validate_python(dict_data) + return TypeAdapter(NodeGet).validate_python(dict_data) def _get_dynamic_service_get_from( @@ -215,7 +210,7 @@ def _get_dynamic_service_get_from( def _get_node_get_idle() -> NodeGetIdle: - return NodeGetIdleAdapter.validate_python( + return TypeAdapter(NodeGetIdle).validate_python( NodeGetIdle.model_config["json_schema_extra"]["example"] ) diff --git a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py index 9a60347a589..b1dfd7c0d1f 100644 --- a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py +++ b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py @@ -20,12 +20,7 @@ DynamicServiceStart, DynamicServiceStop, ) -from models_library.api_schemas_webserver.projects_nodes import ( - NodeGet, - NodeGetAdapter, - NodeGetIdle, - NodeGetIdleAdapter, -) +from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle from models_library.projects_nodes_io import NodeID from pydantic import NonNegativeInt, TypeAdapter from pytest_mock import MockerFixture @@ -82,7 +77,7 @@ def _get_node_get_with(state: str, node_id: NodeID = _DEFAULT_NODE_ID) -> NodeGe ("service_uuid", f"{node_id}"), ], ) - return NodeGetAdapter.validate_python(dict_data) + return TypeAdapter(NodeGet).validate_python(dict_data) def _get_dynamic_service_get_legacy_with( @@ -127,7 +122,7 @@ def _get_node_get_idle(node_id: NodeID = _DEFAULT_NODE_ID) -> NodeGetIdle: ("service_uuid", f"{node_id}"), ], ) - return NodeGetIdleAdapter.validate_python(dict_data) + return TypeAdapter(NodeGetIdle).validate_python(dict_data) class _ResponseTimeline: From 1a3999c2d23f31ccb2d0e2239049e5eaa32fa081 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 18 Oct 2024 08:40:42 +0200 Subject: [PATCH 08/12] revert adapter pattern --- .../settings-library/src/settings_library/basic_types.py | 5 ++--- .../src/simcore_service_dynamic_scheduler/core/settings.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/settings-library/src/settings_library/basic_types.py b/packages/settings-library/src/settings_library/basic_types.py index 49fef50e9ef..4c275397e1b 100644 --- a/packages/settings-library/src/settings_library/basic_types.py +++ b/packages/settings-library/src/settings_library/basic_types.py @@ -4,9 +4,9 @@ # an extra dependency to a larger models_library (intra-repo library) from enum import Enum -from typing import Annotated, Final, TypeAlias +from typing import Annotated, TypeAlias -from pydantic import Field, StringConstraints, TypeAdapter +from pydantic import Field, StringConstraints # port number range PortInt: TypeAlias = Annotated[int, Field(gt=0, lt=65535)] @@ -14,7 +14,6 @@ # e.g. 'v5' VersionTag: TypeAlias = Annotated[str, StringConstraints(pattern=r"^v\d$")] -VersionTagAdapter: Final[TypeAdapter[VersionTag]] = TypeAdapter(VersionTag) class LogLevel(str, Enum): diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 41dc283a6b1..80f629c7537 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -2,10 +2,10 @@ from functools import cached_property from common_library.pydantic_validators import timedelta_try_convert_str_to_float -from pydantic import AliasChoices, Field, field_validator +from pydantic import AliasChoices, Field, TypeAdapter, field_validator from pydantic_settings import SettingsConfigDict from settings_library.application import BaseApplicationSettings -from settings_library.basic_types import LogLevel, VersionTag, VersionTagAdapter +from settings_library.basic_types import LogLevel, VersionTag from settings_library.director_v2 import DirectorV2Settings from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings @@ -21,7 +21,7 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): # CODE STATICS --------------------------------------------------------- API_VERSION: str = API_VERSION APP_NAME: str = PROJECT_NAME - API_VTAG: VersionTag = VersionTagAdapter.validate_python(API_VTAG) + API_VTAG: VersionTag = TypeAdapter(VersionTag).validate_python(API_VTAG) # RUNTIME ----------------------------------------------------------- From 79384bb6c2f1ae70986b2fc57d70b1a7e663073e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 18 Oct 2024 09:09:32 +0200 Subject: [PATCH 09/12] fixed exception handler --- .../src/servicelib/fastapi/exceptions_utils.py | 4 +++- .../src/simcore_service_dynamic_scheduler/api/rest/routes.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/exceptions_utils.py b/packages/service-library/src/servicelib/fastapi/exceptions_utils.py index d55fc0e0a68..bd5f18448b1 100644 --- a/packages/service-library/src/servicelib/fastapi/exceptions_utils.py +++ b/packages/service-library/src/servicelib/fastapi/exceptions_utils.py @@ -15,9 +15,11 @@ async def http_exception_as_json_response( - request: Request, exc: HTTPException + request: Request, exc: Exception ) -> JSONResponse: + assert isinstance(exc, HTTPException) # nosec assert request # nosec + error = DefaultApiError.from_status_code(exc.status_code) error_detail = error.detail or "" diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/routes.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/routes.py index 2b938584780..8c1d3e21ed8 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/routes.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/routes.py @@ -16,4 +16,4 @@ def setup_rest_api(app: FastAPI): app.include_router(api_router) app.add_exception_handler(Exception, handle_errors_as_500) - app.add_exception_handler(HTTPException, http_exception_as_json_response) # type: ignore[arg-type] + app.add_exception_handler(HTTPException, http_exception_as_json_response) From d5fcc1c0b651920379f8b02106542309ab3b7eb1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 18 Oct 2024 09:12:54 +0200 Subject: [PATCH 10/12] rename --- .../src/common_library/pydantic_validators.py | 6 +++--- .../common-library/tests/test_pydantic_validators.py | 4 ++-- .../agent/src/simcore_service_agent/core/settings.py | 12 ++++++------ .../core/dynamic_services_settings/scheduler.py | 10 +++++----- .../src/simcore_service_director_v2/core/settings.py | 4 ++-- .../core/settings.py | 6 +++--- .../simcore_service_dynamic_sidecar/core/settings.py | 10 +++++----- .../core/settings.py | 6 +++--- .../simcore_service_webserver/projects/settings.py | 4 ++-- .../studies_dispatcher/settings.py | 4 ++-- 10 files changed, 33 insertions(+), 33 deletions(-) diff --git a/packages/common-library/src/common_library/pydantic_validators.py b/packages/common-library/src/common_library/pydantic_validators.py index 33f693a4ba5..60f6219fb13 100644 --- a/packages/common-library/src/common_library/pydantic_validators.py +++ b/packages/common-library/src/common_library/pydantic_validators.py @@ -5,10 +5,10 @@ from pydantic import TypeAdapter, field_validator -def timedelta_try_convert_str_to_float(field: str): +def validate_numeric_string_as_timedelta(field: str): """Transforms a float/int number into a valid datetime as it used to work in the past""" - def _try_convert_str_to_float_or_return( + def _numeric_string_as_timedelta( v: datetime.timedelta | str | float, ) -> datetime.timedelta | str | float: if isinstance(v, str): @@ -32,4 +32,4 @@ def _try_convert_str_to_float_or_return( return v return v - return field_validator(field, mode="before")(_try_convert_str_to_float_or_return) + return field_validator(field, mode="before")(_numeric_string_as_timedelta) diff --git a/packages/common-library/tests/test_pydantic_validators.py b/packages/common-library/tests/test_pydantic_validators.py index 825f986918a..da1ccf95adb 100644 --- a/packages/common-library/tests/test_pydantic_validators.py +++ b/packages/common-library/tests/test_pydantic_validators.py @@ -1,7 +1,7 @@ from datetime import timedelta import pytest -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from faker import Faker from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -15,7 +15,7 @@ class Settings(BaseSettings): APP_NAME: str REQUEST_TIMEOUT: timedelta = Field(default=timedelta(seconds=40)) - _try_convert_request_timeout = timedelta_try_convert_str_to_float( + _validate_request_timeout = validate_numeric_string_as_timedelta( "REQUEST_TIMEOUT" ) diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index 2190d1f3d77..bc713ca3779 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -1,6 +1,6 @@ from datetime import timedelta -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import BootModeEnum, LogLevel from pydantic import AliasChoices, AnyHttpUrl, Field, field_validator from settings_library.base import BaseCustomSettings @@ -76,17 +76,17 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): auto_default_from_env=True, description="settings for service/rabbitmq" ) - _try_convert_agent_volumes_cleanup_interval = timedelta_try_convert_str_to_float( + _validate_agent_volumes_cleanup_interval = validate_numeric_string_as_timedelta( "AGENT_VOLUMES_CLEANUP_INTERVAL" ) - _try_convert_agent_volumes_cleanup_book_keeping_interval = ( - timedelta_try_convert_str_to_float( + _validate_agent_volumes_cleanup_book_keeping_interval = ( + validate_numeric_string_as_timedelta( "AGENT_VOLUMES_CLEANUP_BOOK_KEEPING_INTERVAL" ) ) - _try_convert_agent_volumes_cleanup_remove_volumes_inactive_for = ( - timedelta_try_convert_str_to_float( + _validate_agent_volumes_cleanup_remove_volumes_inactive_for = ( + validate_numeric_string_as_timedelta( "AGENT_VOLUMES_CLEANUP_REMOVE_VOLUMES_INACTIVE_FOR" ) ) diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py index fe7fa7db653..74810cdd101 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py @@ -1,7 +1,7 @@ from datetime import timedelta from typing import Final -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.projects_networks import DockerNetworkName from pydantic import Field, NonNegativeInt, PositiveFloat from settings_library.base import BaseCustomSettings @@ -168,11 +168,11 @@ class DynamicServicesSchedulerSettings(BaseCustomSettings): timedelta(0), description="time to sleep before removing a container" ) - _try_convert_director_v2_dynamic_scheduler_interval = ( - timedelta_try_convert_str_to_float("DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL") + _validate_director_v2_dynamic_scheduler_interval = ( + validate_numeric_string_as_timedelta("DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL") ) - _try_convert_director_v2_dynamic_sidecar_sleep_after_container_removal = ( - timedelta_try_convert_str_to_float( + _validate_director_v2_dynamic_sidecar_sleep_after_container_removal = ( + validate_numeric_string_as_timedelta( "DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL" ) ) diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index bf5006a228f..3c63028747b 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -5,7 +5,7 @@ import datetime from functools import cached_property -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import ( BootModeEnum, BuildTargetEnum, @@ -236,6 +236,6 @@ def _validate_loglevel(cls, value: str) -> str: log_level: str = cls.validate_log_level(value) return log_level - _try_convert_service_tracking_heartbeat = timedelta_try_convert_str_to_float( + _validate_service_tracking_heartbeat = validate_numeric_string_as_timedelta( "SERVICE_TRACKING_HEARTBEAT" ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 80f629c7537..391fb4e3da6 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,7 +1,7 @@ import datetime from functools import cached_property -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from pydantic import AliasChoices, Field, TypeAdapter, field_validator from pydantic_settings import SettingsConfigDict from settings_library.application import BaseApplicationSettings @@ -52,8 +52,8 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): def LOG_LEVEL(self): # noqa: N802 return self.DYNAMIC_SCHEDULER__LOGLEVEL - _try_convert_dynamic_scheduler_stop_service_timeout = ( - timedelta_try_convert_str_to_float("DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT") + _validate_dynamic_scheduler_stop_service_timeout = ( + validate_numeric_string_as_timedelta("DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT") ) @field_validator("DYNAMIC_SCHEDULER__LOGLEVEL") diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index ff015934ef7..957cb2b5ab3 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import cast -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import BootModeEnum, PortInt from models_library.callbacks_mapping import CallbacksMapping from models_library.products import ProductName @@ -39,8 +39,8 @@ class ResourceTrackingSettings(BaseCustomSettings): description="each time the status of the service is propagated", ) - _try_convert_resource_tracking_heartbeat_interval = ( - timedelta_try_convert_str_to_float("RESOURCE_TRACKING_HEARTBEAT_INTERVAL") + _validate_resource_tracking_heartbeat_interval = ( + validate_numeric_string_as_timedelta("RESOURCE_TRACKING_HEARTBEAT_INTERVAL") ) @@ -200,8 +200,8 @@ def are_prometheus_metrics_enabled(self) -> bool: def _check_log_level(cls, value): return cls.validate_log_level(value) - _try_convert_dynamic_sidecar_telemetry_disk_usage_monitor_interval = ( - timedelta_try_convert_str_to_float( + _validate_dynamic_sidecar_telemetry_disk_usage_monitor_interval = ( + validate_numeric_string_as_timedelta( "DYNAMIC_SIDECAR_TELEMETRY_DISK_USAGE_MONITOR_INTERVAL" ) ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index 6a09cfa12cd..b4348a00763 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -1,7 +1,7 @@ import datetime from functools import cached_property -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import BootModeEnum from pydantic import Field, PositiveInt, validator from settings_library.base import BaseCustomSettings @@ -116,8 +116,8 @@ class ApplicationSettings(MinimalApplicationSettings): auto_default_from_env=True, description="settings for opentelemetry tracing" ) - _try_convert_resource_usage_tracker_missed_heartbeat_interval_sec = ( - timedelta_try_convert_str_to_float( + _validate_resource_usage_tracker_missed_heartbeat_interval_sec = ( + validate_numeric_string_as_timedelta( "RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC" ) ) diff --git a/services/web/server/src/simcore_service_webserver/projects/settings.py b/services/web/server/src/simcore_service_webserver/projects/settings.py index 2ab2f4f7e30..8a46b8def4f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/settings.py +++ b/services/web/server/src/simcore_service_webserver/projects/settings.py @@ -1,7 +1,7 @@ from datetime import timedelta from aiohttp import web -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from pydantic import ByteSize, Field, NonNegativeInt, parse_obj_as from settings_library.base import BaseCustomSettings @@ -24,7 +24,7 @@ class ProjectsSettings(BaseCustomSettings): description="interval after which services need to be idle in order to be considered inactive", ) - _try_convert_projects_inactivity_interval = timedelta_try_convert_str_to_float( + _validate_projects_inactivity_interval = validate_numeric_string_as_timedelta( "PROJECTS_INACTIVITY_INTERVAL" ) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py index b64d573fa90..a79c4865f12 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py @@ -2,7 +2,7 @@ from typing import Any, ClassVar from aiohttp import web -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from pydantic import ByteSize, HttpUrl, parse_obj_as, validator from pydantic.fields import Field from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY @@ -51,7 +51,7 @@ def is_login_required(self): """ return not self.STUDIES_ACCESS_ANONYMOUS_ALLOWED - _try_convert_studies_guest_account_lifetime = timedelta_try_convert_str_to_float( + _validate_studies_guest_account_lifetime = validate_numeric_string_as_timedelta( "STUDIES_GUEST_ACCOUNT_LIFETIME" ) From 1ccfddb128efdd1beec1ff3c3bbfdbf3fa9ee141 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 18 Oct 2024 10:47:23 +0200 Subject: [PATCH 11/12] debug in ci --- .../tests/unit/service_tracker/test__tracker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py index 59739ddf8f6..f3c3375f881 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py @@ -87,6 +87,7 @@ async def test_tracker_listing(tracker: Tracker, item_count: NonNegativeInt) -> response = await tracker.all() for key in response: assert isinstance(key, NodeID) + assert len(response) == len(data_to_insert) == item_count assert response == data_to_insert From 5cbe4f2442962e4e6b8c416958ebec526b5fdd8b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 18 Oct 2024 11:00:15 +0200 Subject: [PATCH 12/12] disable monitor to avoid interference in tests --- .../tests/unit/service_tracker/test__tracker.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py index f3c3375f881..20293f343b5 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py @@ -7,6 +7,7 @@ from fastapi import FastAPI from models_library.projects_nodes_io import NodeID from pydantic import NonNegativeInt +from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.utils import logged_gather from settings_library.redis import RedisSettings @@ -24,8 +25,17 @@ ] +@pytest.fixture +def disable_monitor_task(mocker: MockerFixture) -> None: + mocker.patch( + "simcore_service_dynamic_scheduler.services.status_monitor._monitor.Monitor._worker_start_get_status_requests", + autospec=True, + ) + + @pytest.fixture def app_environment( + disable_monitor_task: None, disable_rabbitmq_setup: None, disable_deferred_manager_setup: None, disable_notifier_setup: None, @@ -87,7 +97,6 @@ async def test_tracker_listing(tracker: Tracker, item_count: NonNegativeInt) -> response = await tracker.all() for key in response: assert isinstance(key, NodeID) - assert len(response) == len(data_to_insert) == item_count assert response == data_to_insert