Merge branch 'master' into mai/error-models

Author: pcrespov

.env-devel

@@ -333,7 +333,7 @@ LOGIN_2FA_REQUIRED=0
 LOGIN_ACCOUNT_DELETION_RETENTION_DAYS=31
 LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0
 LOGIN_REGISTRATION_INVITATION_REQUIRED=0
-PROJECTS_INACTIVITY_INTERVAL=20
+PROJECTS_INACTIVITY_INTERVAL=00:00:20
 PROJECTS_TRASH_RETENTION_DAYS=7
 PROJECTS_MAX_COPY_SIZE_BYTES=30Gib
 PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES=5

packages/common-library/src/common_library/json_serialization.py

@@ -23,7 +23,7 @@
 from uuid import UUID
 
 import orjson
-from pydantic import NameEmail, SecretBytes, SecretStr
+from pydantic import AnyHttpUrl, AnyUrl, HttpUrl, NameEmail, SecretBytes, SecretStr
 from pydantic_core import Url
 from pydantic_extra_types.color import Color
 
@@ -62,6 +62,8 @@ def decimal_encoder(dec_value: Decimal) -> int | float:
 
 
 ENCODERS_BY_TYPE: dict[type[Any], Callable[[Any], Any]] = {
+    AnyHttpUrl: str,
+    AnyUrl: str,
     bytes: lambda o: o.decode(),
     Color: str,
     datetime.date: isoformat,
@@ -73,6 +75,7 @@ def decimal_encoder(dec_value: Decimal) -> int | float:
     frozenset: list,
     deque: list,
     GeneratorType: list,
+    HttpUrl: str,
     IPv4Address: str,
     IPv4Interface: str,
     IPv4Network: str,

packages/common-library/tests/test_json_serialization.py

@@ -15,7 +15,7 @@
     json_loads,
 )
 from faker import Faker
-from pydantic import Field, TypeAdapter
+from pydantic import AnyHttpUrl, AnyUrl, BaseModel, Field, HttpUrl, TypeAdapter
 from pydantic.json import pydantic_encoder
 
 
@@ -95,3 +95,18 @@ def test_compatiblity_with_json_interface(
 
     # NOTE: cannot compare dumps directly because orjson compacts it more
     assert json_loads(orjson_dump) == json_loads(json_dump)
+
+
+def test_serialized_model_with_urls(faker: Faker):
+    # See: https://github.com/ITISFoundation/osparc-simcore/pull/6852
+    class M(BaseModel):
+        any_http_url: AnyHttpUrl
+        any_url: AnyUrl
+        http_url: HttpUrl
+
+    obj = M(
+        any_http_url=faker.url(),
+        any_url=faker.url(),
+        http_url=faker.url(),
+    )
+    json_dumps(obj)

packages/models-library/src/models_library/utils/_original_fastapi_encoders.py

@@ -8,7 +8,7 @@
 from enum import Enum
 from pathlib import PurePath
 from types import GeneratorType
-from typing import Any, Callable, Union
+from typing import Any, Callable, Union, get_origin
 
 from common_library.json_serialization import ENCODERS_BY_TYPE
 from pydantic import BaseModel
@@ -28,7 +28,8 @@ def generate_encoders_by_class_tuples(
         tuple
     )
     for type_, encoder in type_encoder_map.items():
-        encoders_by_class_tuples[encoder] += (type_,)
+        if get_origin(type_) is not Annotated:
+            encoders_by_class_tuples[encoder] += (type_,)
     return encoders_by_class_tuples
 
 

services/api-server/src/simcore_service_api_server/exceptions/handlers/_log_streaming_errors.py

@@ -4,7 +4,7 @@
 
 from ..log_streaming_errors import (
     LogStreamerNotRegisteredError,
-    LogStreamerRegistionConflictError,
+    LogStreamerRegistrationConflictError,
     LogStreamingBaseError,
 )
 from ._utils import create_error_json_response
@@ -18,7 +18,7 @@ async def log_handling_error_handler(request: Request, exc: Exception) -> JSONRe
     status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR
     if isinstance(exc, LogStreamerNotRegisteredError):
         status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
-    elif isinstance(exc, LogStreamerRegistionConflictError):
+    elif isinstance(exc, LogStreamerRegistrationConflictError):
         status_code = status.HTTP_409_CONFLICT
 
     return create_error_json_response(msg, status_code=status_code)

services/api-server/src/simcore_service_api_server/exceptions/log_streaming_errors.py

@@ -9,5 +9,5 @@ class LogStreamerNotRegisteredError(LogStreamingBaseError):
     msg_template = "{msg}"
 
 
-class LogStreamerRegistionConflictError(LogStreamingBaseError):
+class LogStreamerRegistrationConflictError(LogStreamingBaseError):
     msg_template = "A stream was already connected to {job_id}. Only a single stream can be connected at the time"

services/api-server/src/simcore_service_api_server/services/log_streaming.py

@@ -16,7 +16,7 @@
 from ..exceptions.backend_errors import BaseBackEndError
 from ..exceptions.log_streaming_errors import (
     LogStreamerNotRegisteredError,
-    LogStreamerRegistionConflictError,
+    LogStreamerRegistrationConflictError,
 )
 from ..models.schemas.errors import ErrorGet
 from ..models.schemas.jobs import JobID, JobLog
@@ -70,7 +70,7 @@ async def _distribute_logs(self, data: bytes):
 
     async def register(self, job_id: JobID, queue: Queue[JobLog]):
         if job_id in self._log_streamers:
-            raise LogStreamerRegistionConflictError(job_id=job_id)
+            raise LogStreamerRegistrationConflictError(job_id=job_id)
         self._log_streamers[job_id] = queue
         await self._rabbit_client.add_topics(
             LoggerRabbitMessage.get_channel_name(), topics=[f"{job_id}.*"]
@@ -126,26 +126,24 @@ async def log_generator(self) -> AsyncIterable[str]:
                 except TimeoutError:
                     done = await self._project_done()
 
-        except BaseBackEndError as exc:
-            _logger.info("%s", f"{exc}")
-            yield ErrorGet(errors=[f"{exc}"]).model_dump_json() + _NEW_LINE
+        except (BaseBackEndError, LogStreamerRegistrationConflictError) as exc:
+            error_msg = f"{exc}"
+
+            _logger.info("%s: %s", exc.code, error_msg)
+            yield ErrorGet(errors=[error_msg]).model_dump_json() + _NEW_LINE
+
         except Exception as exc:  # pylint: disable=W0718
             error_code = create_error_code(exc)
-            user_error_msg = (
-                MSG_INTERNAL_ERROR_USER_FRIENDLY_TEMPLATE + f" [{error_code}]"
-            )
+            error_msg = MSG_INTERNAL_ERROR_USER_FRIENDLY_TEMPLATE + f" [{error_code}]"
 
             _logger.exception(
                 **create_troubleshotting_log_kwargs(
-                    user_error_msg,
+                    error_msg,
                     error=exc,
                     error_code=error_code,
                 )
             )
-            yield ErrorGet(
-                errors=[
-                    MSG_INTERNAL_ERROR_USER_FRIENDLY_TEMPLATE + f" (OEC: {error_code})"
-                ]
-            ).model_dump_json() + _NEW_LINE
+            yield ErrorGet(errors=[error_msg]).model_dump_json() + _NEW_LINE
+
         finally:
             await self._log_distributor.deregister(self._job_id)

services/api-server/tests/unit/test_services_rabbitmq.py

@@ -45,7 +45,7 @@
 from simcore_service_api_server.services.log_streaming import (
     LogDistributor,
     LogStreamer,
-    LogStreamerRegistionConflictError,
+    LogStreamerRegistrationConflictError,
 )
 from tenacity import AsyncRetrying, retry_if_not_exception_type, stop_after_delay
 
@@ -219,7 +219,7 @@ async def _(job_log: JobLog):
         pass
 
     await log_distributor.register(project_id, _)
-    with pytest.raises(LogStreamerRegistionConflictError):
+    with pytest.raises(LogStreamerRegistrationConflictError):
         await log_distributor.register(project_id, _)
     await log_distributor.deregister(project_id)
 

services/dynamic-scheduler/tests/unit/test_cli.py

@@ -1,8 +1,10 @@
 # pylint:disable=unused-argument
 
 import os
+import traceback
 
 import pytest
+from click.testing import Result
 from pytest_simcore.helpers.monkeypatch_envs import load_dotenv, setenvs_from_dict
 from pytest_simcore.helpers.typing_env import EnvVarsDict
 from simcore_service_dynamic_scheduler._meta import API_VERSION
@@ -11,20 +13,26 @@
 from typer.testing import CliRunner
 
 
+def _format_cli_error(result: Result) -> str:
+    assert result.exception
+    tb_message = "\n".join(traceback.format_tb(result.exception.__traceback__))
+    return f"Below exception was raised by the cli:\n{tb_message}"
+
+
 def test_cli_help_and_version(cli_runner: CliRunner):
     # simcore-service-dynamic-scheduler --help
     result = cli_runner.invoke(cli_main, "--help")
-    assert result.exit_code == os.EX_OK, result.output
+    assert result.exit_code == os.EX_OK, _format_cli_error(result)
 
     result = cli_runner.invoke(cli_main, "--version")
-    assert result.exit_code == os.EX_OK, result.output
+    assert result.exit_code == os.EX_OK, _format_cli_error(result)
     assert result.stdout.strip() == API_VERSION
 
 
 def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch):
     # simcore-service-dynamic-scheduler echo-dotenv
     result = cli_runner.invoke(cli_main, "echo-dotenv")
-    assert result.exit_code == os.EX_OK, result.output
+    assert result.exit_code == os.EX_OK, _format_cli_error(result)
 
     environs = load_dotenv(result.stdout)
 
@@ -33,10 +41,25 @@ def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch):
         ApplicationSettings.create_from_envs()
 
 
-def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict):
-    # simcore-service-dynamic-scheduler settings --show-secrets --as-json
-    result = cli_runner.invoke(cli_main, ["settings", "--show-secrets", "--as-json"])
-    assert result.exit_code == os.EX_OK, result.output
+def test_list_settings(
+    cli_runner: CliRunner, app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch
+):
+    with monkeypatch.context() as patch:
+        setenvs_from_dict(
+            patch,
+            {
+                **app_environment,
+                "DYNAMIC_SCHEDULER_TRACING": "{}",
+                "TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT": "http://replace-with-opentelemetry-collector",
+                "TRACING_OPENTELEMETRY_COLLECTOR_PORT": "4318",
+            },
+        )
+
+        # simcore-service-dynamic-scheduler settings --show-secrets --as-json
+        result = cli_runner.invoke(
+            cli_main, ["settings", "--show-secrets", "--as-json"]
+        )
+        assert result.exit_code == os.EX_OK, _format_cli_error(result)
 
     print(result.output)
     settings = ApplicationSettings(result.output)

services/web/server/src/simcore_service_webserver/projects/settings.py

@@ -1,7 +1,6 @@
-from datetime import timedelta
+import datetime as dt
 
 from aiohttp import web
-from common_library.pydantic_validators import validate_numeric_string_as_timedelta
 from pydantic import ByteSize, Field, NonNegativeInt, TypeAdapter
 from settings_library.base import BaseCustomSettings
 
@@ -19,14 +18,11 @@ class ProjectsSettings(BaseCustomSettings):
         description="defines the number of dynamic services in a project that can be started concurrently (a value of 0 will disable it)",
     )
 
-    PROJECTS_INACTIVITY_INTERVAL: timedelta = Field(
-        default=timedelta(seconds=20),
+    PROJECTS_INACTIVITY_INTERVAL: dt.timedelta = Field(
+        default=dt.timedelta(seconds=20),
         description="interval after which services need to be idle in order to be considered inactive",
     )
 
-    _validate_projects_inactivity_interval = validate_numeric_string_as_timedelta(
-        "PROJECTS_INACTIVITY_INTERVAL"
-    )
     PROJECTS_TRASH_RETENTION_DAYS: NonNegativeInt = Field(
         default=7, description="Trashed items will be deleted after this time"
     )