From 42d8edad19fe771d83bae0b2c6da78c7e35ce73f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Sat, 5 Oct 2024 13:12:27 +0200 Subject: [PATCH 1/9] some changes --- .../src/models_library/services_creation.py | 1 + .../src/models_library/services_types.py | 17 +- services/dynamic-sidecar/.env-devel | 2 +- services/dynamic-sidecar/openapi.json | 295 +++++++++++------- .../api/containers.py | 4 +- .../core/application.py | 2 +- .../core/settings.py | 2 +- .../modules/long_running_tasks.py | 58 ++-- services/dynamic-sidecar/tests/conftest.py | 7 +- .../tests/unit/test_api_containers.py | 14 +- .../test_api_containers_long_running_tasks.py | 8 +- .../tests/unit/test_api_prometheus_metrics.py | 6 +- 12 files changed, 251 insertions(+), 165 deletions(-) diff --git a/packages/models-library/src/models_library/services_creation.py b/packages/models-library/src/models_library/services_creation.py index c1c2c5172fcd..996ad5f746d2 100644 --- a/packages/models-library/src/models_library/services_creation.py +++ b/packages/models-library/src/models_library/services_creation.py @@ -30,6 +30,7 @@ class CreateServiceMetricsAdditionalParams(BaseModel): "wallet_name": "a private wallet for me", "pricing_plan_id": 1, "pricing_unit_id": 1, + "pricing_unit_cost_id": 1, "pricing_unit_detail_id": 1, "product_name": "osparc", "simcore_user_agent": "undefined", diff --git a/packages/models-library/src/models_library/services_types.py b/packages/models-library/src/models_library/services_types.py index 87559eff9939..192d52dcd0d7 100644 --- a/packages/models-library/src/models_library/services_types.py +++ b/packages/models-library/src/models_library/services_types.py @@ -2,7 +2,8 @@ from uuid import uuid4 import arrow -from pydantic import StringConstraints +from models_library.basic_types import ConstrainedStr +from pydantic import StringConstraints, TypeAdapter from .basic_regex import PROPERTY_KEY_RE, SIMPLE_VERSION_RE from .services_regex import ( @@ -19,9 +20,13 @@ ServiceKey: TypeAlias = Annotated[str, StringConstraints(pattern=SERVICE_KEY_RE)] -ServiceKeyEncoded: TypeAlias = Annotated[str, StringConstraints(pattern=SERVICE_ENCODED_KEY_RE)] +ServiceKeyEncoded: TypeAlias = Annotated[ + str, StringConstraints(pattern=SERVICE_ENCODED_KEY_RE) +] -DynamicServiceKey: TypeAlias = Annotated[str, StringConstraints(pattern=DYNAMIC_SERVICE_KEY_RE)] +DynamicServiceKey: TypeAlias = Annotated[ + str, StringConstraints(pattern=DYNAMIC_SERVICE_KEY_RE) +] ComputationalServiceKey: TypeAlias = Annotated[ str, StringConstraints(pattern=COMPUTATIONAL_SERVICE_KEY_RE) @@ -30,7 +35,7 @@ ServiceVersion: TypeAlias = Annotated[str, StringConstraints(pattern=SIMPLE_VERSION_RE)] -class RunID(str): +class RunID(ConstrainedStr): """ Used to assign a unique identifier to the run of a service. @@ -41,8 +46,6 @@ class RunID(str): and gives the osparc-agent an opportunity to back it up. """ - __slots__ = () - @classmethod def create(cls) -> "RunID": # NOTE: there was a legacy version of this RunID @@ -52,4 +55,4 @@ def create(cls) -> "RunID": # '1690203099_0ac3ed64-665b-42d2-95f7-e59e0db34242' utc_int_timestamp: int = arrow.utcnow().int_timestamp run_id_format = f"{utc_int_timestamp}_{uuid4()}" - return cls(run_id_format) + return TypeAdapter(cls).validate_python(run_id_format) diff --git a/services/dynamic-sidecar/.env-devel b/services/dynamic-sidecar/.env-devel index 39b904af4d36..b760c0df3bff 100644 --- a/services/dynamic-sidecar/.env-devel +++ b/services/dynamic-sidecar/.env-devel @@ -24,7 +24,7 @@ DY_SIDECAR_USER_SERVICES_HAVE_INTERNET_ACCESS=false # DOCKER REGISTRY DY_DEPLOYMENT_REGISTRY_SETTINGS='{"REGISTRY_AUTH":"false","REGISTRY_USER":"test","REGISTRY_PW":"test","REGISTRY_SSL":"false"}' -S3_ENDPOINT=http://145.456.25.54:12345 +S3_ENDPOINT=http://111.222.111.222:12345 S3_ACCESS_KEY=mocked S3_REGION=mocked S3_SECRET_KEY=mocked diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index d3bfc8ab2435..32e2d8afd583 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -120,16 +120,16 @@ "operationId": "containers_docker_inspect_v1_containers_get", "parameters": [ { - "description": "if True only show the status of the container", + "name": "only_status", + "in": "query", "required": false, "schema": { "type": "boolean", - "title": "Only Status", "description": "if True only show the status of the container", - "default": false + "default": false, + "title": "Only Status" }, - "name": "only_status", - "in": "query" + "description": "if True only show the status of the container" } ], "responses": { @@ -166,14 +166,14 @@ "summary": "Starts the containers as defined in ContainerCreate by:\n- cleaning up resources from previous runs if any\n- starting the containers\n\nProgress may be obtained through URL\nProcess may be cancelled through URL", "operationId": "create_service_containers_task_v1_containers_post", "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ContainersCreate" } } - }, - "required": true + } }, "responses": { "202": { @@ -213,7 +213,15 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ActivityInfo" + "anyOf": [ + { + "$ref": "#/components/schemas/ActivityInfo" + }, + { + "type": "null" + } + ], + "title": "Response Get Containers Activity V1 Containers Activity Get" } } } @@ -231,16 +239,17 @@ "operationId": "get_container_logs_v1_containers__id__logs_get", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Id" - }, - "name": "id", - "in": "path" + } }, { - "description": "Only return logs since this time, as a UNIX timestamp", + "name": "since", + "in": "query", "required": false, "schema": { "type": "integer", @@ -248,11 +257,11 @@ "description": "Only return logs since this time, as a UNIX timestamp", "default": 0 }, - "name": "since", - "in": "query" + "description": "Only return logs since this time, as a UNIX timestamp" }, { - "description": "Only return logs before this time, as a UNIX timestamp", + "name": "until", + "in": "query", "required": false, "schema": { "type": "integer", @@ -260,11 +269,11 @@ "description": "Only return logs before this time, as a UNIX timestamp", "default": 0 }, - "name": "until", - "in": "query" + "description": "Only return logs before this time, as a UNIX timestamp" }, { - "description": "Enabling this parameter will include timestamps in logs", + "name": "timestamps", + "in": "query", "required": false, "schema": { "type": "boolean", @@ -272,8 +281,7 @@ "description": "Enabling this parameter will include timestamps in logs", "default": false }, - "name": "timestamps", - "in": "query" + "description": "Enabling this parameter will include timestamps in logs" } ], "responses": { @@ -282,10 +290,10 @@ "content": { "application/json": { "schema": { + "type": "array", "items": { "type": "string" }, - "type": "array", "title": "Response Get Container Logs V1 Containers Id Logs Get" } } @@ -320,15 +328,15 @@ "operationId": "get_containers_name_v1_containers_name_get", "parameters": [ { - "description": "JSON encoded dictionary. FastAPI does not allow for dict as type in query parameters", + "name": "filters", + "in": "query", "required": true, "schema": { "type": "string", - "title": "Filters", - "description": "JSON encoded dictionary. FastAPI does not allow for dict as type in query parameters" + "description": "JSON encoded dictionary. FastAPI does not allow for dict as type in query parameters", + "title": "Filters" }, - "name": "filters", - "in": "query" + "description": "JSON encoded dictionary. FastAPI does not allow for dict as type in query parameters" } ], "responses": { @@ -369,13 +377,13 @@ "operationId": "inspect_container_v1_containers__id__get", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Id" - }, - "name": "id", - "in": "path" + } } ], "responses": { @@ -486,24 +494,24 @@ "operationId": "attach_container_to_network_v1_containers__id__networks_attach_post", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Id" - }, - "name": "id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AttachContainerToNetworkItem" } } - }, - "required": true + } }, "responses": { "204": { @@ -531,24 +539,24 @@ "operationId": "detach_container_from_network_v1_containers__id__networks_detach_post", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Id" - }, - "name": "id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DetachContainerFromNetworkItem" } } - }, - "required": true + } }, "responses": { "204": { @@ -666,10 +674,17 @@ "content": { "application/json": { "schema": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Port Keys" } } @@ -711,10 +726,17 @@ "content": { "application/json": { "schema": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Port Keys" } } @@ -798,23 +820,23 @@ "operationId": "put_volume_state_v1_volumes__id__put", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "$ref": "#/components/schemas/VolumeCategory" - }, - "name": "id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PutVolumeItem" } } - }, - "required": true + } }, "responses": { "204": { @@ -873,7 +895,14 @@ "default": true }, "error_message": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Error Message", "description": "in case of error this gets set" } @@ -909,8 +938,7 @@ "GPU", "MPI" ], - "title": "BootMode", - "description": "An enumeration." + "title": "BootMode" }, "ContainersComposeSpec": { "properties": { @@ -956,25 +984,60 @@ "CreateServiceMetricsAdditionalParams": { "properties": { "wallet_id": { - "type": "integer", - "exclusiveMinimum": true, - "title": "Wallet Id", - "minimum": 0 + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Wallet Id" }, "wallet_name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Wallet Name" }, "pricing_plan_id": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Pricing Plan Id" }, "pricing_unit_id": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Pricing Unit Id" }, "pricing_unit_cost_id": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Pricing Unit Cost Id" }, "product_name": { @@ -1008,9 +1071,6 @@ "title": "Service Version" }, "service_resources": { - "additionalProperties": { - "$ref": "#/components/schemas/ImageResources" - }, "type": "object", "title": "Service Resources" }, @@ -1021,6 +1081,11 @@ }, "type": "object", "required": [ + "wallet_id", + "wallet_name", + "pricing_plan_id", + "pricing_unit_id", + "pricing_unit_cost_id", "product_name", "simcore_user_agent", "user_email", @@ -1033,20 +1098,21 @@ ], "title": "CreateServiceMetricsAdditionalParams", "example": { - "wallet_id": 1, - "wallet_name": "a private wallet for me", + "node_name": "the service of a lifetime _ *!", "pricing_plan_id": 1, - "pricing_unit_id": 1, + "pricing_unit_cost_id": 1, "pricing_unit_detail_id": 1, + "pricing_unit_id": 1, "product_name": "osparc", - "simcore_user_agent": "undefined", - "user_email": "test@test.com", "project_name": "_!New Study", - "node_name": "the service of a lifetime _ *!", + "service_additional_metadata": {}, "service_key": "simcore/services/dynamic/test", - "service_version": "0.0.1", "service_resources": {}, - "service_additional_metadata": {} + "service_version": "0.0.1", + "simcore_user_agent": "undefined", + "user_email": "test@test.com", + "wallet_id": 1, + "wallet_name": "a private wallet for me" } }, "DetachContainerFromNetworkItem": { @@ -1095,6 +1161,7 @@ "$ref": "#/components/schemas/BootMode" }, "type": "array", + "title": "Boot Modes", "description": "describe how a service shall be booted, using CPU, MPI, openMP or GPU", "default": [ "CPU" @@ -1110,6 +1177,14 @@ "example": { "image": "simcore/service/dynamic/pretty-intense:1.0.0", "resources": { + "AIRAM": { + "limit": 1, + "reservation": 1 + }, + "ANY_resource": { + "limit": "some_value", + "reservation": "some_value" + }, "CPU": { "limit": 4, "reservation": 0.1 @@ -1121,14 +1196,6 @@ "VRAM": { "limit": 1, "reservation": 1 - }, - "AIRAM": { - "limit": 1, - "reservation": 1 - }, - "ANY_resource": { - "limit": "some_value", - "reservation": "some_value" } } } @@ -1222,7 +1289,14 @@ "ServiceOutput": { "properties": { "displayOrder": { - "type": "number", + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], "title": "Displayorder", "description": "DEPRECATED: new display order is taken from the item position. This will be removed.", "deprecated": true @@ -1230,14 +1304,12 @@ "label": { "type": "string", "title": "Label", - "description": "short name for the property", - "example": "Age" + "description": "short name for the property" }, "description": { "type": "string", "title": "Description", - "description": "description of the property", - "example": "Age in seconds since 1970" + "description": "description of the property" }, "type": { "type": "string", @@ -1246,32 +1318,51 @@ "description": "data type expected on this input glob matching for data type is allowed" }, "contentSchema": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Contentschema", "description": "jsonschema of this input/output. Required when type='ref_contentSchema'" }, "fileToKeyMap": { - "additionalProperties": { - "type": "string", - "pattern": "^[-_a-zA-Z0-9]+$" - }, - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Filetokeymap", "description": "Place the data associated with the named keys in files" }, "unit": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Unit", "description": "Units, when it refers to a physical quantity", "deprecated": true }, "widget": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Widget" + }, + { + "type": "null" } ], - "title": "Widget", "description": "custom widget to use instead of the default one determined from the data-type", "deprecated": true } @@ -1283,8 +1374,7 @@ "description", "type" ], - "title": "ServiceOutput", - "description": "Base class for service input/outputs" + "title": "ServiceOutput" }, "Structure": { "properties": { @@ -1389,11 +1479,7 @@ "Widget": { "properties": { "type": { - "allOf": [ - { - "$ref": "#/components/schemas/WidgetType" - } - ], + "$ref": "#/components/schemas/WidgetType", "description": "type of the property" }, "details": { @@ -1422,8 +1508,7 @@ "TextArea", "SelectBox" ], - "title": "WidgetType", - "description": "An enumeration." + "title": "WidgetType" } } } diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 71ec22cec919..656f7be0e98b 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -12,7 +12,7 @@ ActivityInfo, ActivityInfoOrNone, ) -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from servicelib.fastapi.requests_decorators import cancel_on_disconnect from ..core.docker_utils import docker_client @@ -175,7 +175,7 @@ async def get_containers_activity( try: return TypeAdapter(ActivityInfo).validate_json(inactivity_response) - except json.JSONDecodeError: + except ValidationError: _logger.warning( "Could not parse command result '%s' as '%s'", inactivity_response, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index f5910ffbffee..bde49c4bed87 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -129,7 +129,7 @@ def create_base_app() -> FastAPI: # settings settings = ApplicationSettings.create_from_envs() setup_logger(settings) - logger.debug(settings.json(indent=2)) + logger.debug(settings.model_dump_json(indent=2)) # minimal app = FastAPI( diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index 1ad855e9f1d1..b07428ad9c47 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -155,7 +155,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): RABBIT_SETTINGS: RabbitSettings = Field(auto_default_from_env=True) DY_DEPLOYMENT_REGISTRY_SETTINGS: RegistrySettings = Field() - DY_DOCKER_HUB_REGISTRY_SETTINGS: RegistrySettings | None = Field() + DY_DOCKER_HUB_REGISTRY_SETTINGS: RegistrySettings | None = Field(default=None) RESOURCE_TRACKING: ResourceTrackingSettings = Field(auto_default_from_env=True) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py index 9c13cfc54170..615a1b0b2b95 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py @@ -164,7 +164,7 @@ async def task_create_service_containers( app: FastAPI, application_health: ApplicationHealth, ) -> list[str]: - progress.update(message="validating service spec", percent=ProgressPercent(0)) + progress.update(message="validating service spec", percent=0) assert shared_store.compose_spec # nosec @@ -178,7 +178,7 @@ async def task_create_service_containers( result = await docker_compose_rm(shared_store.compose_spec, settings) _raise_for_errors(result, "rm") - progress.update(message="pulling images", percent=ProgressPercent(0.01)) + progress.update(message="pulling images", percent=0.01) await post_sidecar_log_message( app, "pulling service images", log_level=logging.INFO ) @@ -187,17 +187,13 @@ async def task_create_service_containers( app, "service images ready", log_level=logging.INFO ) - progress.update( - message="creating and starting containers", percent=ProgressPercent(0.90) - ) + progress.update(message="creating and starting containers", percent=0.90) await post_sidecar_log_message( app, "starting service containers", log_level=logging.INFO ) await _retry_docker_compose_create(shared_store.compose_spec, settings) - progress.update( - message="ensure containers are started", percent=ProgressPercent(0.95) - ) + progress.update(message="ensure containers are started", percent=0.95) compose_start_result = await _retry_docker_compose_start( shared_store.compose_spec, settings ) @@ -279,9 +275,7 @@ async def _send_resource_tracking_stop(platform_status: SimcorePlatformStatus): await send_service_stopped(app, simcore_platform_status) try: - progress.update( - message="running docker-compose-down", percent=ProgressPercent(0.1) - ) + progress.update(message="running docker-compose-down", percent=0.1) await run_before_shutdown_actions( shared_store, settings.DY_SIDECAR_CALLBACKS_MAPPING.before_shutdown @@ -294,13 +288,11 @@ async def _send_resource_tracking_stop(platform_status: SimcorePlatformStatus): result = await _retry_docker_compose_down(shared_store.compose_spec, settings) _raise_for_errors(result, "down") - progress.update(message="stopping logs", percent=ProgressPercent(0.9)) + progress.update(message="stopping logs", percent=0.9) for container_name in shared_store.container_names: await stop_log_fetching(app, container_name) - progress.update( - message="removing pending resources", percent=ProgressPercent(0.95) - ) + progress.update(message="removing pending resources", percent=0.95) result = await docker_compose_rm(shared_store.compose_spec, settings) _raise_for_errors(result, "rm") except Exception: @@ -314,7 +306,7 @@ async def _send_resource_tracking_stop(platform_status: SimcorePlatformStatus): async with shared_store: shared_store.compose_spec = None shared_store.container_names = [] - progress.update(message="done", percent=ProgressPercent(0.99)) + progress.update(message="done", percent=0.99) async def _restore_state_folder( @@ -354,7 +346,7 @@ async def task_restore_state( # NOTE: this implies that the legacy format will always be decompressed # until it is not removed. - progress.update(message="Downloading state", percent=ProgressPercent(0.05)) + progress.update(message="Downloading state", percent=0.05) state_paths = list(mounted_volumes.disk_state_paths_iter()) await post_sidecar_log_message( app, @@ -384,7 +376,7 @@ async def task_restore_state( await post_sidecar_log_message( app, "Finished state downloading", log_level=logging.INFO ) - progress.update(message="state restored", percent=ProgressPercent(0.99)) + progress.update(message="state restored", percent=0.99) async def _save_state_folder( @@ -421,7 +413,7 @@ async def task_save_state( If a legacy archive is detected, it will be removed after saving the new format. """ - progress.update(message="starting state save", percent=ProgressPercent(0.0)) + progress.update(message="starting state save", percent=0.0) state_paths = list(mounted_volumes.disk_state_paths_iter()) async with ProgressBarData( num_steps=len(state_paths), @@ -447,7 +439,7 @@ async def task_save_state( ) await post_sidecar_log_message(app, "Finished state saving", log_level=logging.INFO) - progress.update(message="finished state saving", percent=ProgressPercent(0.99)) + progress.update(message="finished state saving", percent=0.99) async def task_ports_inputs_pull( @@ -462,12 +454,12 @@ async def task_ports_inputs_pull( _logger.info("Received request to pull inputs but was ignored") return 0 - progress.update(message="starting inputs pulling", percent=ProgressPercent(0.0)) + progress.update(message="starting inputs pulling", percent=0.0) port_keys = [] if port_keys is None else port_keys await post_sidecar_log_message( app, f"Pulling inputs for {port_keys}", log_level=logging.INFO ) - progress.update(message="pulling inputs", percent=ProgressPercent(0.1)) + progress.update(message="pulling inputs", percent=0.1) async with ProgressBarData( num_steps=1, progress_report_cb=functools.partial( @@ -492,7 +484,7 @@ async def task_ports_inputs_pull( await post_sidecar_log_message( app, "Finished pulling inputs", log_level=logging.INFO ) - progress.update(message="finished inputs pulling", percent=ProgressPercent(0.99)) + progress.update(message="finished inputs pulling", percent=0.99) return int(transferred_bytes) @@ -502,7 +494,7 @@ async def task_ports_outputs_pull( mounted_volumes: MountedVolumes, app: FastAPI, ) -> int: - progress.update(message="starting outputs pulling", percent=ProgressPercent(0.0)) + progress.update(message="starting outputs pulling", percent=0.0) port_keys = [] if port_keys is None else port_keys await post_sidecar_log_message( app, f"Pulling output for {port_keys}", log_level=logging.INFO @@ -528,14 +520,14 @@ async def task_ports_outputs_pull( await post_sidecar_log_message( app, "Finished pulling outputs", log_level=logging.INFO ) - progress.update(message="finished outputs pulling", percent=ProgressPercent(0.99)) + progress.update(message="finished outputs pulling", percent=0.99) return int(transferred_bytes) async def task_ports_outputs_push( progress: TaskProgress, outputs_manager: OutputsManager, app: FastAPI ) -> None: - progress.update(message="starting outputs pushing", percent=ProgressPercent(0.0)) + progress.update(message="starting outputs pushing", percent=0.0) await post_sidecar_log_message( app, f"waiting for outputs {outputs_manager.outputs_context.file_type_port_keys} to be pushed", @@ -547,7 +539,7 @@ async def task_ports_outputs_push( await post_sidecar_log_message( app, "finished outputs pushing", log_level=logging.INFO ) - progress.update(message="finished outputs pushing", percent=ProgressPercent(0.99)) + progress.update(message="finished outputs pushing", percent=0.99) async def task_containers_restart( @@ -562,9 +554,7 @@ async def task_containers_restart( # or some other state, the service will get shutdown, to prevent this # blocking status while containers are being restarted. async with app.state.container_restart_lock: - progress.update( - message="starting containers restart", percent=ProgressPercent(0.0) - ) + progress.update(message="starting containers restart", percent=0.0) if shared_store.compose_spec is None: msg = "No spec for docker compose command was found" raise RuntimeError(msg) @@ -572,20 +562,20 @@ async def task_containers_restart( for container_name in shared_store.container_names: await stop_log_fetching(app, container_name) - progress.update(message="stopped log fetching", percent=ProgressPercent(0.1)) + progress.update(message="stopped log fetching", percent=0.1) result = await docker_compose_restart(shared_store.compose_spec, settings) _raise_for_errors(result, "restart") - progress.update(message="containers restarted", percent=ProgressPercent(0.8)) + progress.update(message="containers restarted", percent=0.8) for container_name in shared_store.container_names: await start_log_fetching(app, container_name) - progress.update(message="started log fetching", percent=ProgressPercent(0.9)) + progress.update(message="started log fetching", percent=0.9) await post_sidecar_log_message( app, "Service was restarted please reload the UI", log_level=logging.INFO ) await post_event_reload_iframe(app) - progress.update(message="started log fetching", percent=ProgressPercent(0.99)) + progress.update(message="started log fetching", percent=0.99) diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 97760f43fdbb..24c5558283a2 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -20,7 +20,7 @@ from models_library.services_creation import CreateServiceMetricsAdditionalParams from models_library.users import UserID from models_library.utils.json_serialization import json_dumps -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import ( EnvVarsDict, @@ -330,11 +330,10 @@ def mock_stop_heart_beat_task(mocker: MockerFixture) -> AsyncMock: @pytest.fixture def mock_metrics_params(faker: Faker) -> CreateServiceMetricsAdditionalParams: - return parse_obj_as( - CreateServiceMetricsAdditionalParams, + return TypeAdapter(CreateServiceMetricsAdditionalParams).validate_python( CreateServiceMetricsAdditionalParams.model_config["json_schema_extra"][ "example" - ], + ] ) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index d8f32426cbc6..6d3a76e17de3 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -750,7 +750,10 @@ async def test_containers_activity_command_failed( ): response = await test_client.get(f"/{API_VTAG}/containers/activity") assert response.status_code == 200, response.text - assert response.json() == ActivityInfo(seconds_inactive=_INACTIVE_FOR_LONG_TIME) + assert ( + response.json() + == ActivityInfo(seconds_inactive=_INACTIVE_FOR_LONG_TIME).model_dump() + ) async def test_containers_activity_no_inactivity_defined( @@ -773,7 +776,7 @@ def mock_inactive_since_command_response( ) -> None: mocker.patch( "simcore_service_dynamic_sidecar.api.containers.run_command_in_container", - return_value=activity_response.json(), + return_value=activity_response.model_dump_json(), ) @@ -786,7 +789,7 @@ async def test_containers_activity_inactive_since( ): response = await test_client.get(f"/{API_VTAG}/containers/activity") assert response.status_code == 200, response.text - assert response.json() == activity_response + assert response.json() == activity_response.model_dump() @pytest.fixture @@ -805,4 +808,7 @@ async def test_containers_activity_unexpected_response( ): response = await test_client.get(f"/{API_VTAG}/containers/activity") assert response.status_code == 200, response.text - assert response.json() == ActivityInfo(seconds_inactive=_INACTIVE_FOR_LONG_TIME) + assert ( + response.json() + == ActivityInfo(seconds_inactive=_INACTIVE_FOR_LONG_TIME).model_dump() + ) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py index 1adde00e2065..ef2086d8304b 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py @@ -187,7 +187,7 @@ async def httpx_async_client( # crete dir here async with AsyncClient( app=app, - base_url=backend_url, + base_url=f"{backend_url}", headers={"Content-Type": "application/json"}, ) as client: yield client @@ -388,9 +388,11 @@ async def test_create_containers_task( mock_metrics_params: CreateServiceMetricsAdditionalParams, shared_store: SharedStore, ) -> None: - last_progress_message: tuple[str, float] | None = None + last_progress_message: tuple[ProgressMessage, ProgressPercent | None] | None = None - async def create_progress(message: str, percent: float, _: TaskId) -> None: + async def create_progress( + message: ProgressMessage, percent: ProgressPercent | None, _: TaskId + ) -> None: nonlocal last_progress_message last_progress_message = (message, percent) print(message, percent) diff --git a/services/dynamic-sidecar/tests/unit/test_api_prometheus_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_prometheus_metrics.py index ba7089203a5b..a790e34e53cb 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_prometheus_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_prometheus_metrics.py @@ -14,7 +14,7 @@ from httpx import AsyncClient from models_library.callbacks_mapping import CallbacksMapping from models_library.services_creation import CreateServiceMetricsAdditionalParams -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.fastapi.long_running_tasks.client import ( Client, @@ -59,7 +59,7 @@ async def app(mock_rabbitmq_envs: EnvVarsDict, app: FastAPI) -> AsyncIterable[Fa @pytest.fixture def backend_url() -> AnyHttpUrl: - return parse_obj_as(AnyHttpUrl, "http://backgroud.testserver.io") + return TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io") @pytest.fixture @@ -71,7 +71,7 @@ async def httpx_async_client( ) -> AsyncIterable[AsyncClient]: async with AsyncClient( app=app, - base_url=backend_url, + base_url=f"{backend_url}", headers={"Content-Type": "application/json"}, ) as client: yield client From f45222b57226446a884b077c8e6d0b28f3c4da6d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 8 Oct 2024 09:33:26 +0200 Subject: [PATCH 2/9] fixed failing test --- .../tests/unit/test_api_containers_long_running_tasks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py index ef2086d8304b..6c475781d5b2 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py @@ -16,6 +16,7 @@ from aiodocker.containers import DockerContainer from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI from fastapi.routing import APIRoute from httpx import AsyncClient @@ -24,7 +25,7 @@ ProgressPercent, ) from models_library.services_creation import CreateServiceMetricsAdditionalParams -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from servicelib.fastapi.long_running_tasks.client import ( @@ -157,7 +158,9 @@ def compose_spec(request: pytest.FixtureRequest) -> str: @pytest.fixture def backend_url() -> AnyHttpUrl: - return parse_obj_as(AnyHttpUrl, "http://backgroud.testserver.io") + return TypeAdapter(AnyHttpUrlLegacy).validate_python( + "http://backgroud.testserver.io" + ) @pytest.fixture From 7a32427e6c4b29189d0a9f6d394824bf4fe1bd76 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 8 Oct 2024 12:47:58 +0200 Subject: [PATCH 3/9] fixed broken tests --- .../modules/outputs/_manager.py | 11 +++++------ .../unit/test_api_containers_long_running_tasks.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py index 307f8b3d9337..61c2d5130e22 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py @@ -6,11 +6,11 @@ from datetime import timedelta from functools import partial +from common_library.errors_classes import OsparcErrorMixin from fastapi import FastAPI from models_library.basic_types import IDStr from models_library.rabbitmq_messages import ProgressType from pydantic import PositiveFloat -from pydantic.errors import PydanticErrorMixin from servicelib import progress_bar from servicelib.background_task import start_periodic_task, stop_periodic_task from servicelib.logging_utils import log_catch, log_context @@ -26,12 +26,11 @@ async def _cancel_task(task: Task, task_cancellation_timeout_s: PositiveFloat) -> None: task.cancel() - with suppress(CancelledError): - with log_catch(logger, reraise=False): - await wait((task,), timeout=task_cancellation_timeout_s) + with suppress(CancelledError), log_catch(logger, reraise=False): + await wait((task,), timeout=task_cancellation_timeout_s) -class UploadPortsFailed(PydanticErrorMixin, RuntimeError): +class UploadPortsFailedError(OsparcErrorMixin, RuntimeError): code: str = "dynamic_sidecar.outputs_manager.failed_while_uploading" msg_template: str = "Failed while uploading: failures={failures}" @@ -247,7 +246,7 @@ async def wait_for_all_uploads_to_finish(self) -> None: True for v in self._last_upload_error_tracker.values() if v is not None ) if any_failed_upload: - raise UploadPortsFailed(failures=self._last_upload_error_tracker) + raise UploadPortsFailedError(failures=self._last_upload_error_tracker) def setup_outputs_manager(app: FastAPI) -> None: diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py index 6c475781d5b2..d7f0a0ac9f65 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers_long_running_tasks.py @@ -200,7 +200,7 @@ async def httpx_async_client( def client( app: FastAPI, httpx_async_client: AsyncClient, backend_url: AnyHttpUrl ) -> Client: - return Client(app=app, async_client=httpx_async_client, base_url=backend_url) + return Client(app=app, async_client=httpx_async_client, base_url=f"{backend_url}") @pytest.fixture From 9349ea436010ae1832e8ae7721ca21810b970f0a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 8 Oct 2024 12:55:45 +0200 Subject: [PATCH 4/9] fixed broken tests --- .../tests/unit/test_api_prometheus_metrics.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_prometheus_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_prometheus_metrics.py index a790e34e53cb..e8206dc47a90 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_prometheus_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_prometheus_metrics.py @@ -10,6 +10,7 @@ import pytest from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI, status from httpx import AsyncClient from models_library.callbacks_mapping import CallbacksMapping @@ -59,7 +60,9 @@ async def app(mock_rabbitmq_envs: EnvVarsDict, app: FastAPI) -> AsyncIterable[Fa @pytest.fixture def backend_url() -> AnyHttpUrl: - return TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io") + return TypeAdapter(AnyHttpUrlLegacy).validate_python( + "http://backgroud.testserver.io" + ) @pytest.fixture @@ -81,7 +84,7 @@ async def httpx_async_client( def client( app: FastAPI, httpx_async_client: AsyncClient, backend_url: AnyHttpUrl ) -> Client: - return Client(app=app, async_client=httpx_async_client, base_url=backend_url) + return Client(app=app, async_client=httpx_async_client, base_url=f"{backend_url}") @pytest.fixture From 165bcdbab7b58b69c04e6462b03cbe77b2663ac0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 8 Oct 2024 12:56:44 +0200 Subject: [PATCH 5/9] fixed broken import --- .../tests/unit/test_modules_outputs_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_modules_outputs_manager.py b/services/dynamic-sidecar/tests/unit/test_modules_outputs_manager.py index 40a3db6d3f94..c15190c9705b 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_outputs_manager.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_outputs_manager.py @@ -28,7 +28,7 @@ ) from simcore_service_dynamic_sidecar.modules.outputs._manager import ( OutputsManager, - UploadPortsFailed, + UploadPortsFailedError, _PortKeyTracker, setup_outputs_manager, ) @@ -230,7 +230,7 @@ async def test_recovers_after_raising_error( assert await outputs_manager._port_key_tracker.no_tracked_ports() is False await asyncio.sleep(outputs_manager.task_monitor_interval_s * 10) - with pytest.raises(UploadPortsFailed) as exec_info: + with pytest.raises(UploadPortsFailedError) as exec_info: await outputs_manager.wait_for_all_uploads_to_finish() assert set(exec_info.value.failures.keys()) == set(port_keys) | set( From 17c192153d95c56d881e2e35c947884c0e65a083 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 8 Oct 2024 14:33:04 +0200 Subject: [PATCH 6/9] fixed some broken tests --- .../pydantic_networks_extension.py | 4 +- .../core/external_dependencies.py | 4 +- .../core/settings.py | 6 +++ .../core/storage.py | 8 +-- .../models/shared_store.py | 13 ++++- .../modules/service_liveness.py | 4 +- .../tests/unit/test_api_volumes.py | 4 +- .../unit/test_api_workflow_service_metrics.py | 7 ++- .../tests/unit/test_models_shared_store.py | 50 +++++++++++++------ 9 files changed, 71 insertions(+), 29 deletions(-) diff --git a/packages/common-library/src/common_library/pydantic_networks_extension.py b/packages/common-library/src/common_library/pydantic_networks_extension.py index 75095c39e8da..3b87db20e8a1 100644 --- a/packages/common-library/src/common_library/pydantic_networks_extension.py +++ b/packages/common-library/src/common_library/pydantic_networks_extension.py @@ -1,6 +1,6 @@ from typing import Annotated, TypeAlias -from pydantic import AfterValidator, AnyHttpUrl +from pydantic import AfterValidator, AnyHttpUrl, AnyUrl def _strip_last_slash(url: str) -> str: @@ -10,3 +10,5 @@ def _strip_last_slash(url: str) -> str: AnyHttpUrlLegacy: TypeAlias = Annotated[ str, AnyHttpUrl, AfterValidator(_strip_last_slash) ] + +AnyUrlLegacy: TypeAlias = Annotated[str, AnyUrl, AfterValidator(_strip_last_slash)] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/external_dependencies.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/external_dependencies.py index 12696fe13f08..278f29e7ad19 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/external_dependencies.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/external_dependencies.py @@ -1,5 +1,5 @@ +from common_library.errors_classes import OsparcErrorMixin from fastapi import FastAPI -from pydantic.errors import PydanticErrorMixin from servicelib.utils import logged_gather from .postgres import wait_for_postgres_liveness @@ -8,7 +8,7 @@ from .storage import wait_for_storage_liveness -class CouldNotReachExternalDependenciesError(PydanticErrorMixin, Exception): +class CouldNotReachExternalDependenciesError(OsparcErrorMixin, Exception): msg_template: str = ( "Could not start because the following external dependencies failed: {failed}" ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index b07428ad9c47..4344f0575d66 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -4,6 +4,9 @@ from pathlib import Path from typing import cast +from common_library.pydantic_settings_validators import ( + validate_timedelta_in_legacy_mode, +) from models_library.basic_types import BootModeEnum, PortInt from models_library.callbacks_mapping import CallbacksMapping from models_library.products import ProductName @@ -30,6 +33,9 @@ class ResourceTrackingSettings(BaseCustomSettings): default=DEFAULT_RESOURCE_USAGE_HEARTBEAT_INTERVAL, description="each time the status of the service is propagated", ) + _validate_legacy_heartbeat_interval = validate_timedelta_in_legacy_mode( + "RESOURCE_TRACKING_HEARTBEAT_INTERVAL" + ) class SystemMonitorSettings(BaseCustomSettings): diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/storage.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/storage.py index 9118711a5732..639ee0dd8102 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/storage.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/storage.py @@ -4,7 +4,7 @@ from fastapi import FastAPI, status from httpx import AsyncClient -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from servicelib.logging_utils import log_context from settings_library.node_ports import StorageAuthSettings @@ -33,8 +33,10 @@ def _get_auth_or_none(storage_auth_settings: StorageAuthSettings) -> _AuthTuple def _get_url(storage_auth_settings: StorageAuthSettings) -> str: - url: str = parse_obj_as(AnyUrl, f"{storage_auth_settings.api_base_url}/") - return url + url: AnyUrl = TypeAdapter(AnyUrl).validate_python( + f"{storage_auth_settings.api_base_url}/" + ) + return f"{url}" async def _is_storage_responsive(storage_auth_settings: StorageAuthSettings) -> bool: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py index 0745f595fa83..17ec4fac8f12 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py @@ -66,6 +66,17 @@ class SharedStore(_StoreMixin): default_factory=dict, description="persist the state of each volume" ) + def __eq__(self, other: object) -> bool: + return all( + getattr(self, n, None) == getattr(other, n, None) + for n in ( + "compose_spec", + "container_names", + "original_to_container_names", + "volume_states", + ) + ) + async def _setup_initial_volume_states(self) -> None: async with self: for category, status in [ @@ -93,7 +104,7 @@ async def init_from_disk( async with aiofiles.open(shared_store_dir / store_file_name) as data_file: file_content = await data_file.read() - obj = cls.parse_raw(file_content) + obj = cls.model_validate_json(file_content) obj.post_init(shared_store_dir) return obj diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/service_liveness.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/service_liveness.py index 78976b53a868..33c0083861f5 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/service_liveness.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/service_liveness.py @@ -4,7 +4,7 @@ from datetime import timedelta from typing import Final -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin from tenacity import AsyncRetrying, RetryCallState, TryAgain from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed @@ -16,7 +16,7 @@ _DEFAULT_TIMEOUT_INTERVAL: Final[timedelta] = timedelta(seconds=30) -class CouldNotReachServiceError(PydanticErrorMixin, Exception): +class CouldNotReachServiceError(OsparcErrorMixin, Exception): msg_template: str = "Could not contact service '{service_name}' at '{endpoint}'. Look above for details." diff --git a/services/dynamic-sidecar/tests/unit/test_api_volumes.py b/services/dynamic-sidecar/tests/unit/test_api_volumes.py index fe396d002ad4..40eab12336a3 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_volumes.py +++ b/services/dynamic-sidecar/tests/unit/test_api_volumes.py @@ -56,6 +56,4 @@ async def test_volumes_state_saved_error( ) assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY, response.text json_response = response.json() - assert ( - invalid_volume_category not in json_response["detail"][0]["ctx"]["enum_values"] - ) + assert invalid_volume_category not in json_response["detail"][0]["ctx"]["expected"] diff --git a/services/dynamic-sidecar/tests/unit/test_api_workflow_service_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_workflow_service_metrics.py index 22400cb0e809..2183df44b6f8 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_workflow_service_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_workflow_service_metrics.py @@ -16,6 +16,7 @@ from aiodocker.utils import clean_filters from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI from httpx import AsyncClient from models_library.generated_models.docker_rest_api import ContainerState @@ -28,7 +29,7 @@ SimcorePlatformStatus, ) from models_library.services_creation import CreateServiceMetricsAdditionalParams -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.fastapi.long_running_tasks.client import ( @@ -79,7 +80,9 @@ def compose_spec(raw_compose_spec: dict[str, Any]) -> str: @pytest.fixture def backend_url() -> AnyHttpUrl: - return parse_obj_as(AnyHttpUrl, "http://backgroud.testserver.io") + return TypeAdapter(AnyHttpUrlLegacy).validate_python( + "http://backgroud.testserver.io" + ) @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_models_shared_store.py b/services/dynamic-sidecar/tests/unit/test_models_shared_store.py index c72a8bdb85a8..2c2b474a0290 100644 --- a/services/dynamic-sidecar/tests/unit/test_models_shared_store.py +++ b/services/dynamic-sidecar/tests/unit/test_models_shared_store.py @@ -6,11 +6,12 @@ from pathlib import Path from typing import Any +import arrow import pytest from async_asgi_testclient import TestClient from fastapi import FastAPI from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from servicelib.utils import logged_gather from simcore_service_dynamic_sidecar.core import application @@ -53,17 +54,17 @@ def mock_docker_compose(mocker: MockerFixture) -> None: {"volume_states": {}}, { "volume_states": { - VolumeCategory.OUTPUTS: parse_obj_as( - VolumeState, {"status": VolumeStatus.CONTENT_NO_SAVE_REQUIRED} + VolumeCategory.OUTPUTS: TypeAdapter(VolumeState).validate_python( + {"status": VolumeStatus.CONTENT_NO_SAVE_REQUIRED} ), - VolumeCategory.INPUTS: parse_obj_as( - VolumeState, {"status": VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED} + VolumeCategory.INPUTS: TypeAdapter(VolumeState).validate_python( + {"status": VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED} ), - VolumeCategory.STATES: parse_obj_as( - VolumeState, {"status": VolumeStatus.CONTENT_WAS_SAVED} + VolumeCategory.STATES: TypeAdapter(VolumeState).validate_python( + {"status": VolumeStatus.CONTENT_WAS_SAVED} ), - VolumeCategory.SHARED_STORE: parse_obj_as( - VolumeState, {"status": VolumeStatus.CONTENT_NO_SAVE_REQUIRED} + VolumeCategory.SHARED_STORE: TypeAdapter(VolumeState).validate_python( + {"status": VolumeStatus.CONTENT_NO_SAVE_REQUIRED} ), } }, @@ -88,7 +89,18 @@ async def test_shared_store_updates( # check the contes of the file should be the same as the shared_store's assert store_file_path.exists() is True - assert shared_store == SharedStore.parse_raw(store_file_path.read_text()) + + def _normalize_datetimes(shared_store: SharedStore) -> None: + for state in shared_store.volume_states.values(): + state.last_changed = arrow.get(state.last_changed.isoformat()).datetime + + shared_store_from_file = SharedStore.model_validate_json( + store_file_path.read_text() + ) + _normalize_datetimes(shared_store) + _normalize_datetimes(shared_store_from_file) + + assert shared_store == shared_store_from_file async def test_no_concurrency_with_parallel_writes( @@ -119,12 +131,20 @@ async def test_init_from_disk_with_legacy_data_format(project_tests_dir: Path): # if file is missing it correctly loaded the storage_file assert (MOCKS_DIR / STORE_FILE_NAME).exists() is False - # ensure object objects are compatible - parsed_legacy_format = json.loads(disk_shared_store.json()) + def _normalize_datetimes(data: dict[str, Any]) -> None: + for state in data["volume_states"].values(): + state["last_changed"] = arrow.get( + state["last_changed"] + ).datetime.isoformat() - assert parsed_legacy_format == json.loads( - (MOCKS_DIR / LEGACY_SHARED_STORE).read_text() - ) + # ensure objects are compatible + parsed_legacy_format = json.loads(disk_shared_store.model_dump_json()) + load_raw_from_disk = json.loads((MOCKS_DIR / LEGACY_SHARED_STORE).read_text()) + + _normalize_datetimes(parsed_legacy_format) + _normalize_datetimes(load_raw_from_disk) + + assert parsed_legacy_format == load_raw_from_disk async def test_init_from_disk_no_file_present(tmp_path: Path): From c92edc6a5748edafd28c84ae975fe564fa23935e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 8 Oct 2024 14:44:09 +0200 Subject: [PATCH 7/9] fixed broken tests --- .../tests/unit/test_modules_system_monitor__disk_usage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py b/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py index 2c4c3c6a125d..77875c48c38e 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py @@ -13,7 +13,7 @@ from models_library.projects_nodes_io import NodeID from models_library.users import UserID from psutil._common import sdiskusage -from pydantic import ByteSize +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from simcore_service_dynamic_sidecar.modules.system_monitor._disk_usage import ( DiskUsageMonitor, @@ -59,7 +59,7 @@ def _get_entry(mock: Mock, *, index: int) -> dict[Path, DiskUsage]: def _get_byte_size(byte_size_as_str: str) -> ByteSize: - return ByteSize.validate(byte_size_as_str) + return TypeAdapter(ByteSize).validate_python(byte_size_as_str) def _get_mocked_disk_usage(byte_size_as_str: str) -> DiskUsage: From b6b40d1d3efd457cd949369dd7001fcf4852763e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 8 Oct 2024 14:52:44 +0200 Subject: [PATCH 8/9] refactor --- .../core/reserved_space.py | 4 ++-- .../core/settings.py | 6 +++--- .../modules/outputs/_directory_utils.py | 6 +++--- .../modules/outputs/_event_filter.py | 6 +++--- .../modules/user_services_preferences/_db.py | 6 ++++-- .../test_modules_long_running_tasks.py | 9 ++++----- .../test_modules_user_services_preferences.py | 10 ++++++---- .../tests/unit/test_core_reserved_space.py | 6 ++++-- .../unit/test_modules_outputs_event_filter.py | 11 +++++++---- .../tests/unit/test_modules_outputs_watcher.py | 18 +++++++++--------- ...ser_services_preferences_user_preference.py | 6 ++++-- 11 files changed, 49 insertions(+), 39 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/reserved_space.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/reserved_space.py index 945aaccb8e5b..e43946f5375f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/reserved_space.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/reserved_space.py @@ -3,14 +3,14 @@ from typing import Final from fastapi import FastAPI -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from .settings import ApplicationSettings _RESERVED_DISK_SPACE_NAME: Final[Path] = Path( "/tmp/reserved_disk_space" # nosec # noqa: S108 ) -_DEFAULT_CHUNK_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "8k") +_DEFAULT_CHUNK_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("8k") def _write_random_binary_file( diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index 4344f0575d66..3ff8e8b7f53d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -14,7 +14,7 @@ from models_library.projects_nodes_io import NodeID from models_library.services import DynamicServiceKey, RunID, ServiceVersion from models_library.users import UserID -from pydantic import ByteSize, Field, PositiveInt, parse_obj_as, validator +from pydantic import ByteSize, Field, PositiveInt, TypeAdapter, field_validator from settings_library.aws_s3_cli import AwsS3CliSettings from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings @@ -106,7 +106,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) DYNAMIC_SIDECAR_RESERVED_SPACE_SIZE: ByteSize = Field( - parse_obj_as(ByteSize, "10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), description=( "Disk space reserve when the dy-sidecar is started. Can be freed at " "any time via an API call. Main reason to free this disk space is " @@ -171,7 +171,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): def are_prometheus_metrics_enabled(self) -> bool: return self.DY_SIDECAR_CALLBACKS_MAPPING.metrics is not None - @validator("LOG_LEVEL") + @field_validator("LOG_LEVEL") @classmethod def _check_log_level(cls, value): return cls.validate_log_level(value) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_directory_utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_directory_utils.py index 7cc13922244e..21f07bf15231 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_directory_utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_directory_utils.py @@ -1,7 +1,7 @@ import os from pathlib import Path -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter def get_directory_total_size(path: Path) -> ByteSize: @@ -10,7 +10,7 @@ def get_directory_total_size(path: Path) -> ByteSize: # until we do not hit 1 million it can be ignored # NOTE: file size has no impact on performance if not path.exists(): - return parse_obj_as(ByteSize, 0) + return TypeAdapter(ByteSize).validate_python(0) total = 0 for entry in os.scandir(path): @@ -18,4 +18,4 @@ def get_directory_total_size(path: Path) -> ByteSize: total += entry.stat().st_size elif entry.is_dir(): total += get_directory_total_size(Path(entry.path)) - return parse_obj_as(ByteSize, total) + return TypeAdapter(ByteSize).validate_python(total) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_filter.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_filter.py index 8490f9cd72eb..227358f4960e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_filter.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_filter.py @@ -14,7 +14,7 @@ NonNegativeInt, PositiveFloat, PositiveInt, - parse_obj_as, + TypeAdapter, ) from servicelib.logging_utils import log_context from watchdog.observers.api import DEFAULT_OBSERVER_TIMEOUT @@ -27,8 +27,8 @@ logger = logging.getLogger(__name__) -_1_MB: Final[PositiveInt] = parse_obj_as(ByteSize, "1mib") -_500_MB: Final[PositiveInt] = parse_obj_as(ByteSize, "500mib") +_1_MB: Final[PositiveInt] = TypeAdapter(ByteSize).validate_python("1mib") +_500_MB: Final[PositiveInt] = TypeAdapter(ByteSize).validate_python("500mib") class BaseDelayPolicy(ABC): diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py index f2cc53a0d9df..0d010794e235 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py @@ -6,7 +6,7 @@ from models_library.user_preferences import PreferenceName from models_library.users import UserID from packaging.version import Version -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_postgres_database.utils_user_preferences import ( UserServicesUserPreferencesRepo, ) @@ -73,5 +73,7 @@ async def load_preferences( if payload is None: return - preference = parse_obj_as(preference_class, umsgpack.unpackb(payload)) + preference = TypeAdapter(preference_class).validate_python( + umsgpack.unpackb(payload) + ) await dir_from_bytes(preference.value, user_preferences_path) diff --git a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py index 5e70b0a6f79f..ea0fcac08935 100644 --- a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py @@ -23,7 +23,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from models_library.users import UserID -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_factories import random_project from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict @@ -199,7 +199,7 @@ async def restore_legacy_state_archives( user_id=user_id, store_id=SIMCORE_LOCATION, store_name=None, - s3_object=parse_obj_as(SimcoreS3FileID, s3_path), + s3_object=TypeAdapter(SimcoreS3FileID).validate_python(s3_path), path_to_upload=legacy_archive_zip, io_log_redirect_cb=None, ) @@ -303,9 +303,8 @@ def s3_settings(app_state: AppState) -> S3Settings: @pytest.fixture def bucket_name(app_state: AppState) -> S3BucketName: - return parse_obj_as( - S3BucketName, - app_state.settings.DY_SIDECAR_R_CLONE_SETTINGS.R_CLONE_S3.S3_BUCKET_NAME, + return TypeAdapter(S3BucketName).validate_python( + app_state.settings.DY_SIDECAR_R_CLONE_SETTINGS.R_CLONE_S3.S3_BUCKET_NAME ) diff --git a/services/dynamic-sidecar/tests/integration/test_modules_user_services_preferences.py b/services/dynamic-sidecar/tests/integration/test_modules_user_services_preferences.py index 094b30144047..9be0bbdebbf2 100644 --- a/services/dynamic-sidecar/tests/integration/test_modules_user_services_preferences.py +++ b/services/dynamic-sidecar/tests/integration/test_modules_user_services_preferences.py @@ -14,7 +14,7 @@ from models_library.projects import ProjectID from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from pytest_simcore.helpers.postgres_tools import PostgresTestConfig from simcore_service_dynamic_sidecar.core.application import create_app @@ -46,17 +46,19 @@ def dy_sidecar_user_preferences_path(tmp_path: Path) -> Path: @pytest.fixture def service_key() -> ServiceKey: - return parse_obj_as(ServiceKey, "simcore/services/dynamic/test-service-34") + return TypeAdapter(ServiceKey).validate_python( + "simcore/services/dynamic/test-service-34" + ) @pytest.fixture def service_version() -> ServiceVersion: - return parse_obj_as(ServiceVersion, "1.0.0") + return TypeAdapter(ServiceVersion).validate_python("1.0.0") @pytest.fixture def product_name() -> ProductName: - return parse_obj_as(ProductName, "osparc") + return TypeAdapter(ProductName).validate_python("osparc") @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_core_reserved_space.py b/services/dynamic-sidecar/tests/unit/test_core_reserved_space.py index 123f21864a05..c78b800ce5a3 100644 --- a/services/dynamic-sidecar/tests/unit/test_core_reserved_space.py +++ b/services/dynamic-sidecar/tests/unit/test_core_reserved_space.py @@ -2,7 +2,7 @@ # pylint:disable=unused-argument -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from simcore_service_dynamic_sidecar.core.application import create_base_app from simcore_service_dynamic_sidecar.core.reserved_space import ( @@ -18,7 +18,9 @@ def test_reserved_disk_space_workflow( create_base_app() assert _RESERVED_DISK_SPACE_NAME.exists() - assert _RESERVED_DISK_SPACE_NAME.stat().st_size == parse_obj_as(ByteSize, "10MiB") + assert _RESERVED_DISK_SPACE_NAME.stat().st_size == TypeAdapter( + ByteSize + ).validate_python("10MiB") remove_reserved_disk_space() assert not _RESERVED_DISK_SPACE_NAME.exists() diff --git a/services/dynamic-sidecar/tests/unit/test_modules_outputs_event_filter.py b/services/dynamic-sidecar/tests/unit/test_modules_outputs_event_filter.py index 024d966e424b..95097d115451 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_outputs_event_filter.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_outputs_event_filter.py @@ -7,7 +7,7 @@ from unittest.mock import AsyncMock import pytest -from pydantic import ByteSize, NonNegativeFloat, NonNegativeInt, parse_obj_as +from pydantic import ByteSize, NonNegativeFloat, NonNegativeInt, TypeAdapter from pytest_mock.plugin import MockerFixture from simcore_service_dynamic_sidecar.modules.outputs._context import OutputsContext from simcore_service_dynamic_sidecar.modules.outputs._event_filter import ( @@ -224,8 +224,8 @@ def test_default_delay_policy(): wait_policy = DefaultDelayPolicy() # below items are defined by the default policy - LOWER_BOUND = parse_obj_as(ByteSize, "1mib") - UPPER_BOUND = parse_obj_as(ByteSize, "500mib") + LOWER_BOUND = TypeAdapter(ByteSize).validate_python("1mib") + UPPER_BOUND = TypeAdapter(ByteSize).validate_python("500mib") assert wait_policy.get_min_interval() == 1.0 @@ -237,4 +237,7 @@ def test_default_delay_policy(): assert wait_policy.get_wait_interval(UPPER_BOUND - 1) < 10.0 assert wait_policy.get_wait_interval(UPPER_BOUND) == 10.0 assert wait_policy.get_wait_interval(UPPER_BOUND + 1) == 10.0 - assert wait_policy.get_wait_interval(parse_obj_as(ByteSize, "1Tib")) == 10.0 + assert ( + wait_policy.get_wait_interval(TypeAdapter(ByteSize).validate_python("1Tib")) + == 10.0 + ) diff --git a/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py b/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py index f209e4877a75..4db996bf0308 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py @@ -22,7 +22,7 @@ NonNegativeFloat, NonNegativeInt, PositiveFloat, - parse_obj_as, + TypeAdapter, ) from pytest_mock import MockerFixture from simcore_service_dynamic_sidecar.modules.mounted_fs import MountedVolumes @@ -168,20 +168,20 @@ class FileGenerationInfo: @pytest.fixture( params=[ FileGenerationInfo( - size=parse_obj_as(ByteSize, "100b"), - chunk_size=parse_obj_as(ByteSize, "1b"), + size=TypeAdapter(ByteSize).validate_python("100b"), + chunk_size=TypeAdapter(ByteSize).validate_python("1b"), ), FileGenerationInfo( - size=parse_obj_as(ByteSize, "100kib"), - chunk_size=parse_obj_as(ByteSize, "1kib"), + size=TypeAdapter(ByteSize).validate_python("100kib"), + chunk_size=TypeAdapter(ByteSize).validate_python("1kib"), ), FileGenerationInfo( - size=parse_obj_as(ByteSize, "100mib"), - chunk_size=parse_obj_as(ByteSize, "1mib"), + size=TypeAdapter(ByteSize).validate_python("100mib"), + chunk_size=TypeAdapter(ByteSize).validate_python("1mib"), ), FileGenerationInfo( - size=parse_obj_as(ByteSize, "100mib"), - chunk_size=parse_obj_as(ByteSize, "10mib"), + size=TypeAdapter(ByteSize).validate_python("100mib"), + chunk_size=TypeAdapter(ByteSize).validate_python("10mib"), ), ] ) diff --git a/services/dynamic-sidecar/tests/unit/test_modules_user_services_preferences_user_preference.py b/services/dynamic-sidecar/tests/unit/test_modules_user_services_preferences_user_preference.py index 08b6f3da05ed..9496bfceb784 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_user_services_preferences_user_preference.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_user_services_preferences_user_preference.py @@ -3,7 +3,7 @@ import pytest from models_library.services import ServiceKey -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_service_dynamic_sidecar.modules.user_services_preferences._user_preference import ( get_model_class, ) @@ -11,7 +11,9 @@ @pytest.fixture def service_key() -> ServiceKey: - return parse_obj_as(ServiceKey, "simcore/services/dynamic/test-service-34") + return TypeAdapter(ServiceKey).validate_python( + "simcore/services/dynamic/test-service-34" + ) def test_get_model_class_only_defined_once(service_key: ServiceKey): From 9c764f4729a170c5ff8bd37a4aec287cf0d4dd40 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 14 Oct 2024 15:51:44 +0200 Subject: [PATCH 9/9] fixed settings --- .../core/settings.py | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index 3ff8e8b7f53d..91706d44be7c 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -14,7 +14,14 @@ from models_library.projects_nodes_io import NodeID from models_library.services import DynamicServiceKey, RunID, ServiceVersion from models_library.users import UserID -from pydantic import ByteSize, Field, PositiveInt, TypeAdapter, field_validator +from pydantic import ( + AliasChoices, + ByteSize, + Field, + PositiveInt, + TypeAdapter, + field_validator, +) from settings_library.aws_s3_cli import AwsS3CliSettings from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings @@ -136,7 +143,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=["DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], + validation_alias=AliasChoices( + "DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED" + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) DY_SIDECAR_USER_ID: UserID @@ -150,22 +159,32 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): DY_SIDECAR_PRODUCT_NAME: ProductName | None = None NODE_PORTS_STORAGE_AUTH: StorageAuthSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} + ) + DY_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field( + json_schema_extra={"auto_default_from_env": True} ) - DY_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(auto_default_from_env=True) DY_SIDECAR_AWS_S3_CLI_SETTINGS: AwsS3CliSettings | None = Field( None, description="AWS S3 settings are used for the AWS S3 CLI. If these settings are filled, the AWS S3 CLI is used instead of RClone.", ) - POSTGRES_SETTINGS: PostgresSettings = Field(auto_default_from_env=True) - RABBIT_SETTINGS: RabbitSettings = Field(auto_default_from_env=True) + POSTGRES_SETTINGS: PostgresSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) + RABBIT_SETTINGS: RabbitSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) DY_DEPLOYMENT_REGISTRY_SETTINGS: RegistrySettings = Field() DY_DOCKER_HUB_REGISTRY_SETTINGS: RegistrySettings | None = Field(default=None) - RESOURCE_TRACKING: ResourceTrackingSettings = Field(auto_default_from_env=True) + RESOURCE_TRACKING: ResourceTrackingSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) - SYSTEM_MONITOR_SETTINGS: SystemMonitorSettings = Field(auto_default_from_env=True) + SYSTEM_MONITOR_SETTINGS: SystemMonitorSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) @property def are_prometheus_metrics_enabled(self) -> bool: