Merge branch 'is4481/upgrade-libs' into is4481/upgrade-api-server

Author: giancarloromeo

packages/models-library/src/models_library/utils/pydantic_fields_extension.py

@@ -0,0 +1,23 @@
+from types import UnionType
+from typing import Any, Literal, get_args, get_origin
+
+from pydantic.fields import FieldInfo
+
+
+def get_type(info: FieldInfo) -> Any:
+    field_type = info.annotation
+    if args := get_args(info.annotation):
+        field_type = next(a for a in args if a != type(None))
+    return field_type
+
+
+def is_literal(info: FieldInfo) -> bool:
+    origin = get_origin(info.annotation)
+    return origin is Literal
+
+
+def is_nullable(info: FieldInfo) -> bool:
+    origin = get_origin(info.annotation)  # X | None or Optional[X] will return Union
+    if origin is UnionType:
+        return any(x in get_args(info.annotation) for x in (type(None), Any))
+    return False

packages/models-library/src/models_library/utils/serialization.py

@@ -0,0 +1,32 @@
+from typing import Any
+
+from models_library.utils.pydantic_fields_extension import get_type
+from pydantic import BaseModel, SecretStr
+
+
+def model_dump_with_secrets(
+    settings_obj: BaseModel, show_secrets: bool, **pydantic_export_options
+) -> dict[str, Any]:
+    data = settings_obj.model_dump(**pydantic_export_options)
+
+    for field_name in settings_obj.model_fields:
+        if field_name not in data:
+            continue
+
+        field_data = data[field_name]
+
+        if isinstance(field_data, SecretStr):
+            if show_secrets:
+                data[field_name] = field_data.get_secret_value()  # Expose the raw value
+            else:
+                data[field_name] = str(field_data)
+        elif isinstance(field_data, dict):
+            field_type = get_type(settings_obj.model_fields[field_name])
+            if issubclass(field_type, BaseModel):
+                data[field_name] = model_dump_with_secrets(
+                    field_type.model_validate(field_data),
+                    show_secrets,
+                    **pydantic_export_options,
+                )
+
+    return data

packages/service-library/tests/deferred_tasks/test_deferred_tasks.py

@@ -16,6 +16,8 @@
 import psutil
 import pytest
 from aiohttp.test_utils import unused_port
+from models_library.utils.json_serialization import json_dumps
+from models_library.utils.serialization import model_dump_with_secrets
 from pydantic import NonNegativeFloat, NonNegativeInt
 from pytest_mock import MockerFixture
 from servicelib import redis as servicelib_redis
@@ -24,7 +26,6 @@
 from servicelib.sequences_utils import partition_gen
 from settings_library.rabbit import RabbitSettings
 from settings_library.redis import RedisSettings
-from settings_library.utils_encoders import create_json_encoder_wo_secrets
 from tenacity.asyncio import AsyncRetrying
 from tenacity.retry import retry_if_exception_type
 from tenacity.stop import stop_after_delay
@@ -123,7 +124,6 @@ async def _tcp_command(
 
 def _get_serialization_options() -> dict[str, Any]:
     return {
-        "encoder": create_json_encoder_wo_secrets(RabbitSettings),
         "exclude_defaults": True,
         "exclude_none": True,
         "exclude_unset": True,
@@ -158,8 +158,20 @@ async def start(self) -> None:
         response = await _tcp_command(
             "init-context",
             {
-                "rabbit": self.rabbit_service.model_dump_json(**_get_serialization_options()),
-                "redis": self.redis_service.model_dump_json(**_get_serialization_options()),
+                "rabbit": json_dumps(
+                    model_dump_with_secrets(
+                        self.rabbit_service,
+                        show_secrets=True,
+                        **_get_serialization_options(),
+                    )
+                ),
+                "redis": json_dumps(
+                    model_dump_with_secrets(
+                        self.redis_service,
+                        show_secrets=True,
+                        **_get_serialization_options(),
+                    )
+                ),
                 "max-workers": self.max_workers,
             },
             port=self.remote_process.port,

packages/settings-library/requirements/ci.txt

@@ -12,6 +12,7 @@
 
 # installs this repo's packages
 pytest-simcore @ ../pytest-simcore
+simcore-models-library @ ../models-library
 
 # current module
 simcore-settings-library @ .

packages/settings-library/src/settings_library/base.py

@@ -1,8 +1,12 @@
 import logging
 from functools import cached_property
-from types import UnionType
-from typing import Any, Final, Literal, get_args, get_origin
+from typing import Any, Final, get_origin
 
+from models_library.utils.pydantic_fields_extension import (
+    get_type,
+    is_literal,
+    is_nullable,
+)
 from pydantic import ValidationInfo, field_validator
 from pydantic.fields import FieldInfo
 from pydantic_core import PydanticUndefined, ValidationError
@@ -21,38 +25,19 @@ def __init__(self, errors):
         self.errors = errors
 
 
-def _allows_none(info: FieldInfo) -> bool:
-    origin = get_origin(info.annotation)  # X | None or Optional[X] will return Union
-    if origin is UnionType:
-        return any(x in get_args(info.annotation) for x in (type(None), Any))
-    return False
-
-
-def _get_type(info: FieldInfo) -> Any:
-    field_type = info.annotation
-    if args := get_args(info.annotation):
-        field_type = next(a for a in args if a != type(None))
-    return field_type
-
-
-def _is_literal(info: FieldInfo) -> bool:
-    origin = get_origin(info.annotation)
-    return origin is Literal
-
-
 def _create_settings_from_env(field_name: str, info: FieldInfo):
     # NOTE: Cannot pass only field.type_ because @prepare_field (when this function is called)
     #  this value is still not resolved (field.type_ at that moment has a weak_ref).
     #  Therefore we keep the entire 'field' but MUST be treated here as read-only
 
     def _default_factory():
         """Creates default from sub-settings or None (if nullable)"""
-        field_settings_cls = _get_type(info)
+        field_settings_cls = get_type(info)
         try:
             return field_settings_cls()
 
         except ValidationError as err:
-            if _allows_none(info):
+            if is_nullable(info):
                 # e.g. Optional[PostgresSettings] would warn if defaults to None
                 _logger.warning(
                     _DEFAULTS_TO_NONE_MSG,
@@ -80,9 +65,9 @@ def _parse_none(cls, v, info: ValidationInfo):
         # WARNING: In nullable fields, envs equal to null or none are parsed as None !!
         if (
             info.field_name
-            and _allows_none(cls.model_fields[info.field_name])
+            and is_nullable(cls.model_fields[info.field_name])
             and isinstance(v, str)
-            and v.lower() in ("null", "none")
+            and v.lower() in ("none",)
         ):
             return None
         return v
@@ -93,6 +78,7 @@ def _parse_none(cls, v, info: ValidationInfo):
         frozen=True,
         validate_default=True,
         ignored_types=(cached_property,),
+        env_parse_none_str="null",
     )
 
     @classmethod
@@ -106,13 +92,13 @@ def __pydantic_init_subclass__(cls, **kwargs: Any):
                     "auto_default_from_env", False
                 )
             )
-            field_type = _get_type(field)
+            field_type = get_type(field)
 
             # Avoids issubclass raising TypeError. SEE test_issubclass_type_error_with_pydantic_models
             is_not_composed = (
                 get_origin(field_type) is None
             )  # is not composed as dict[str, Any] or Generic[Base]
-            is_not_literal = not _is_literal(field)
+            is_not_literal = not is_literal(field)
 
             if (
                 is_not_literal

packages/settings-library/src/settings_library/twilio.py

@@ -8,13 +8,16 @@
 
 from typing import Annotated, TypeAlias
 
-from pydantic import Field, StringConstraints, TypeAdapter
+from pydantic import BeforeValidator, Field, StringConstraints, TypeAdapter
 
 from .base import BaseCustomSettings
 
-
 # Based on https://countrycode.org/
-CountryCodeStr: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, pattern=r"^\d{1,4}")]
+CountryCodeStr: TypeAlias = Annotated[
+    str,
+    BeforeValidator(str),
+    StringConstraints(strip_whitespace=True, pattern=r"^\d{1,4}"),
+]
 
 
 class TwilioSettings(BaseCustomSettings):

packages/settings-library/src/settings_library/utils_cli.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 from collections.abc import Callable
@@ -6,12 +7,12 @@
 
 import rich
 import typer
+from models_library.utils.serialization import model_dump_with_secrets
 from pydantic import ValidationError
 from pydantic_settings import BaseSettings
 
 from ._constants import HEADER_STR
 from .base import BaseCustomSettings
-from .utils_encoders import create_json_encoder_wo_secrets
 
 
 def print_as_envfile(
@@ -25,8 +26,9 @@ def print_as_envfile(
     exclude_unset = pydantic_export_options.get("exclude_unset", False)
 
     for name, field in settings_obj.model_fields.items():
-        auto_default_from_env = field.json_schema_extra is not None and field.json_schema_extra.get(
-            "auto_default_from_env", False  # type: ignore[union-attr]
+        auto_default_from_env = (
+            field.json_schema_extra is not None
+            and field.json_schema_extra.get("auto_default_from_env", False)
         )
 
         value = getattr(settings_obj, name)
@@ -39,7 +41,11 @@ def print_as_envfile(
 
         if isinstance(value, BaseSettings):
             if compact:
-                value = f"'{value.model_dump_json(**pydantic_export_options)}'"  # flat
+                value = json.dumps(
+                    model_dump_with_secrets(
+                        value, show_secrets=show_secrets, **pydantic_export_options
+                    )
+                )  # flat
             else:
                 if verbose:
                     typer.echo(f"\n# --- {name} --- ")
@@ -61,9 +67,16 @@ def print_as_envfile(
         typer.echo(f"{name}={value}")
 
 
-def print_as_json(settings_obj, *, compact=False, **pydantic_export_options):
+def print_as_json(
+    settings_obj, *, compact=False, show_secrets, **pydantic_export_options
+):
     typer.echo(
-        settings_obj.model_dump_json(indent=None if compact else 2, **pydantic_export_options)
+        json.dumps(
+            model_dump_with_secrets(
+                settings_obj, show_secrets=show_secrets, **pydantic_export_options
+            ),
+            indent=None if compact else 2,
+        )
     )
 
 
@@ -127,14 +140,14 @@ def settings(
             raise
 
         pydantic_export_options: dict[str, Any] = {"exclude_unset": exclude_unset}
-        if show_secrets:
-            # NOTE: this option is for json-only
-            pydantic_export_options["encoder"] = create_json_encoder_wo_secrets(
-                settings_cls
-            )
 
         if as_json:
-            print_as_json(settings_obj, compact=compact, **pydantic_export_options)
+            print_as_json(
+                settings_obj,
+                compact=compact,
+                show_secrets=show_secrets,
+                **pydantic_export_options,
+            )
         else:
             print_as_envfile(
                 settings_obj,

packages/settings-library/src/settings_library/utils_service.py

@@ -115,7 +115,7 @@ def _compose_url(
 
         # post process parts dict
         kwargs = {}
-        for k, v in parts.items():
+        for k, v in parts.items():  # type: ignore[assignment]
             if isinstance(v, SecretStr):
                 value = v.get_secret_value()
             else: