Upgrade agent service (Pydantic v2) (#6508)

Author: giancarloromeo

services/agent/requirements/_base.txt

@@ -11,6 +11,8 @@ aiohttp==3.8.5
     #   aiodocker
 aiosignal==1.2.0
     # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
 anyio==3.6.2
     # via
     #   httpx
@@ -41,16 +43,10 @@ click==8.1.3
     #   uvicorn
 dnspython==2.2.1
     # via email-validator
-email-validator==1.3.0
+email-validator==2.2.0
     # via pydantic
-fastapi==0.96.0
+fastapi==0.115.0
     # via
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
     #   -r requirements/_base.in
     #   prometheus-fastapi-instrumentator
@@ -106,27 +102,38 @@ prometheus-client==0.19.0
     #   prometheus-fastapi-instrumentator
 prometheus-fastapi-instrumentator==6.1.0
     # via -r requirements/../../../packages/service-library/requirements/_fastapi.in
-pydantic==1.10.2
+pydantic==2.9.2
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
     #   -r requirements/_base.in
     #   fastapi
+    #   pydantic-extra-types
+    #   pydantic-settings
+pydantic-core==2.23.4
+    # via pydantic
+pydantic-extra-types==2.9.0
+    # via -r requirements/../../../packages/models-library/requirements/_base.in
+pydantic-settings==2.5.2
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/settings-library/requirements/_base.in
 pygments==2.15.1
     # via rich
 pyrsistent==0.19.2
     # via jsonschema
 python-dateutil==2.8.2
     # via arrow
 python-dotenv==1.0.0
-    # via -r requirements/_base.in
+    # via
+    #   -r requirements/_base.in
+    #   pydantic-settings
 rich==13.4.2
     # via
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
@@ -143,7 +150,7 @@ sniffio==1.3.0
     # via
     #   anyio
     #   httpx
-starlette==0.27.0
+starlette==0.38.6
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -154,10 +161,12 @@ starlette==0.27.0
     #   fastapi
 typer==0.12.3
     # via -r requirements/../../../packages/settings-library/requirements/_base.in
-typing-extensions==4.4.0
+typing-extensions==4.12.2
     # via
     #   aiodocker
+    #   fastapi
     #   pydantic
+    #   pydantic-core
     #   typer
 uvicorn==0.19.0
     # via

services/agent/requirements/_test.txt

@@ -13,6 +13,10 @@ aiosignal==1.2.0
     # via
     #   -c requirements/_base.txt
     #   aiohttp
+annotated-types==0.7.0
+    # via
+    #   -c requirements/_base.txt
+    #   pydantic
 anyio==3.6.2
     # via
     #   -c requirements/_base.txt
@@ -81,8 +85,6 @@ ecdsa==0.19.0
     #   moto
     #   python-jose
     #   sshpubkeys
-exceptiongroup==1.2.2
-    # via pytest
 faker==27.0.0
     # via -r requirements/_test.in
 flask==3.0.3
@@ -178,11 +180,15 @@ pyasn1==0.6.0
     #   rsa
 pycparser==2.22
     # via cffi
-pydantic==1.10.2
+pydantic==2.9.2
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   aws-sam-translator
+pydantic-core==2.23.4
+    # via
+    #   -c requirements/_base.txt
+    #   pydantic
 pyparsing==3.1.2
     # via moto
 pyrsistent==0.19.2
@@ -257,16 +263,13 @@ sshpubkeys==3.3.1
     # via moto
 sympy==1.13.2
     # via cfn-lint
-tomli==2.0.1
-    # via
-    #   coverage
-    #   pytest
-typing-extensions==4.4.0
+typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
     #   aws-sam-translator
     #   cfn-lint
     #   pydantic
+    #   pydantic-core
 urllib3==2.2.2
     # via
     #   -c requirements/../../../requirements/constraints.txt

services/agent/requirements/_tools.txt

@@ -77,22 +77,12 @@ setuptools==69.2.0
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   pip-tools
-tomli==2.0.1
-    # via
-    #   -c requirements/_test.txt
-    #   black
-    #   build
-    #   mypy
-    #   pip-tools
-    #   pylint
 tomlkit==0.13.2
     # via pylint
-typing-extensions==4.4.0
+typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
-    #   astroid
-    #   black
     #   mypy
 virtualenv==20.26.3
     # via pre-commit

services/agent/src/simcore_service_agent/core/application.py

@@ -38,7 +38,7 @@ def create_app() -> FastAPI:
     # SETTINGS
     settings = ApplicationSettings.create_from_envs()
     _setup_logger(settings)
-    logger.debug(settings.json(indent=2))
+    logger.debug(settings.model_dump_json(indent=2))
 
     assert settings.SC_BOOT_MODE  # nosec
     app = FastAPI(

services/agent/src/simcore_service_agent/core/settings.py

@@ -1,7 +1,7 @@
 from typing import Final
 
 from models_library.basic_types import BootModeEnum, LogLevel
-from pydantic import AnyHttpUrl, Field, NonNegativeInt, validator
+from pydantic import AliasChoices, AnyHttpUrl, Field, NonNegativeInt, field_validator
 from settings_library.base import BaseCustomSettings
 from settings_library.r_clone import S3Provider
 from settings_library.utils_logging import MixinLoggingSettings
@@ -11,16 +11,21 @@
 
 class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     LOGLEVEL: LogLevel = Field(
-        LogLevel.WARNING.value, env=["AGENT_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"]
+        LogLevel.WARNING.value,
+        validation_alias=AliasChoices(
+            "AGENT_LOGLEVEL",
+            "LOG_LEVEL",
+            "LOGLEVEL",
+        ),
     )
     SC_BOOT_MODE: BootModeEnum | None
 
     AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field(
         default=False,
-        env=[
+        validation_alias=AliasChoices(
             "AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED",
             "LOG_FORMAT_LOCAL_DEV_ENABLED",
-        ],
+        ),
         description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!",
     )
     AGENT_VOLUMES_CLEANUP_TARGET_SWARM_STACK_NAME: str = Field(
@@ -47,7 +52,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     )
     AGENT_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True
 
-    @validator("LOGLEVEL")
+    @field_validator("LOGLEVEL")
     @classmethod
     def valid_log_level(cls, value) -> LogLevel:
         return LogLevel(cls.validate_log_level(value))

services/agent/tests/conftest.py

@@ -11,10 +11,11 @@
 import pytest
 import simcore_service_agent
 from aiodocker.volumes import DockerVolume
+from common_library.pydantic_networks_extension import HttpUrlLegacy
 from models_library.basic_types import BootModeEnum
 from models_library.services import RunID
 from moto.server import ThreadedMotoServer
-from pydantic import HttpUrl, parse_obj_as
+from pydantic import HttpUrl, TypeAdapter
 from settings_library.r_clone import S3Provider
 from simcore_service_agent.core.settings import ApplicationSettings
 
@@ -187,9 +188,8 @@ def caplog_info_debug(
 
 
 @pytest.fixture(scope="module")
-def mocked_s3_server_url(mocked_aws_server: ThreadedMotoServer) -> HttpUrl:
+def mocked_s3_server_url(mocked_aws_server: ThreadedMotoServer) -> HttpUrlLegacy:
     # pylint: disable=protected-access
-    return parse_obj_as(
-        HttpUrl,
+    return TypeAdapter(HttpUrlLegacy).validate_python(
         f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}",  # noqa: SLF001
     )

services/agent/tests/unit/test_modules_volumes_cleanup_s3.py

@@ -149,7 +149,7 @@ async def test_store_to_s3(
     )
 
     await _download_files_from_bucket(
-        endpoint=mocked_s3_server_url,
+        endpoint=f"{mocked_s3_server_url}",
         access_key="xxx",
         secret_key="xxx",
         bucket_name=bucket,