continue upgrading

Author: giancarloromeo

packages/postgres-database/src/simcore_postgres_database/models/products.py

@@ -6,7 +6,13 @@
 """
 
 import json
-from typing import Literal, TypedDict
+import sys
+from typing import Literal
+
+if sys.version_info >= (3, 12):
+    from typing import TypedDict
+else:
+    from typing_extensions import TypedDict # See: https://docs.pydantic.dev/2.9/errors/usage_errors/#typed-dict-version
 
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import JSONB

services/web/server/src/simcore_service_webserver/application_settings.py

@@ -3,6 +3,7 @@
 from typing import Any, Final
 
 from aiohttp import web
+from common_library.pydantic_fields_extension import is_nullable
 from models_library.basic_types import (
     BootModeEnum,
     BuildTargetEnum,
@@ -11,8 +12,8 @@
     VersionTag,
 )
 from models_library.utils.change_case import snake_to_camel
-from pydantic import AliasChoices, TypeAdapter, field_validator, model_validator, AnyHttpUrl
-from pydantic.fields import Field, ModelField
+from pydantic import AliasChoices, TypeAdapter, ValidationInfo, field_validator, model_validator, AnyHttpUrl
+from pydantic.fields import Field
 from pydantic.types import PositiveInt
 from settings_library.base import BaseCustomSettings
 from settings_library.email import SMTPSettings
@@ -105,12 +106,12 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     )
     WEBSERVER_LOGLEVEL: LogLevel = Field(
         default=LogLevel.WARNING.value,
-        env=["WEBSERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"],
+        validation_alias=AliasChoices("WEBSERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"),
         # NOTE: suffix '_LOGLEVEL' is used overall
     )
     WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field(
         default=False,
-        env=["WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"],
+        validation_alias=AliasChoices("WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"),
         description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!",
     )
     # TODO: find a better name!?
@@ -168,51 +169,50 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     )
 
     WEBSERVER_DYNAMIC_SCHEDULER: DynamicSchedulerSettings | None = Field(
-        auto_default_from_env=True, description="dynamic-scheduler plugin settings"
+        description="dynamic-scheduler plugin settings", json_schema_extra={"auto_default_from_env": True}
     )
 
-    WEBSERVER_REDIS: RedisSettings | None = Field(auto_default_from_env=True)
+    WEBSERVER_REDIS: RedisSettings | None = Field(json_schema_extra={"auto_default_from_env": True})
 
     WEBSERVER_REST: RestSettings | None = Field(
-        auto_default_from_env=True, description="rest api plugin"
+        description="rest api plugin", json_schema_extra={"auto_default_from_env": True}
     )
 
     WEBSERVER_RESOURCE_MANAGER: ResourceManagerSettings = Field(
-        auto_default_from_env=True, description="resource_manager plugin"
+        description="resource_manager plugin", json_schema_extra={"auto_default_from_env": True}
     )
     WEBSERVER_RESOURCE_USAGE_TRACKER: ResourceUsageTrackerSettings | None = Field(
-        auto_default_from_env=True,
-        description="resource usage tracker service client's plugin",
+        description="resource usage tracker service client's plugin", json_schema_extra={"auto_default_from_env": True}
     )
     WEBSERVER_SCICRUNCH: SciCrunchSettings | None = Field(
-        auto_default_from_env=True, description="scicrunch plugin"
+        description="scicrunch plugin", json_schema_extra={"auto_default_from_env": True}
     )
     WEBSERVER_SESSION: SessionSettings = Field(
-        auto_default_from_env=True, description="session plugin"
+        description="session plugin", json_schema_extra={"auto_default_from_env": True}
     )
 
     WEBSERVER_STATICWEB: StaticWebserverModuleSettings | None = Field(
-        auto_default_from_env=True, description="static-webserver service plugin"
+        description="static-webserver service plugin", json_schema_extra={"auto_default_from_env": True}
     )
     WEBSERVER_STORAGE: StorageSettings | None = Field(
-        auto_default_from_env=True, description="storage service client's plugin"
+        description="storage service client's plugin", json_schema_extra={"auto_default_from_env": True}
     )
     WEBSERVER_STUDIES_DISPATCHER: StudiesDispatcherSettings | None = Field(
-        auto_default_from_env=True, description="studies dispatcher plugin"
+        description="studies dispatcher plugin", json_schema_extra={"auto_default_from_env": True}
     )
 
     WEBSERVER_TRACING: TracingSettings | None = Field(
-        auto_default_from_env=True, description="tracing plugin"
+        description="tracing plugin", json_schema_extra={"auto_default_from_env": True}
     )
 
     WEBSERVER_PROJECTS: ProjectsSettings | None = Field(
-        auto_default_from_env=True, description="projects plugin"
+        description="projects plugin", json_schema_extra={"auto_default_from_env": True}
     )
     WEBSERVER_RABBITMQ: RabbitSettings | None = Field(
-        auto_default_from_env=True, description="rabbitmq plugin"
+        description="rabbitmq plugin", json_schema_extra={"auto_default_from_env": True}
     )
     WEBSERVER_USERS: UsersSettings | None = Field(
-        auto_default_from_env=True, description="users plugin"
+        description="users plugin", json_schema_extra={"auto_default_from_env": True}
     )
 
     # These plugins only require (for the moment) an entry to toggle between enabled/disabled
@@ -268,17 +268,18 @@ def build_vcs_release_url_if_unset(cls, values):
         mode="before"
     )
     @classmethod
-    def enable_only_if_dev_features_allowed(cls, v, values, field: ModelField):
+    def enable_only_if_dev_features_allowed(cls, v, info: ValidationInfo):
         """Ensures that plugins 'under development' get programatically
         disabled if WEBSERVER_DEV_FEATURES_ENABLED=False
         """
-        if values["WEBSERVER_DEV_FEATURES_ENABLED"]:
+        if info.data["WEBSERVER_DEV_FEATURES_ENABLED"]:
             return v
         if v:
             _logger.warning(
-                "%s still under development and will be disabled.", field.name
+                "%s still under development and will be disabled.", info.field_name
             )
-        return None if field.allow_none else False
+
+        return None if info.field_name and is_nullable(cls.model_fields[info.field_name]) else False
 
     @cached_property
     def log_level(self) -> int:

services/web/server/src/simcore_service_webserver/login/settings.py

@@ -36,8 +36,8 @@ class LoginSettings(BaseCustomSettings):
     )
 
     LOGIN_TWILIO: TwilioSettings | None = Field(
-        auto_default_from_env=True,
         description="Twilio service settings. Used to send SMS for 2FA",
+        json_schema_extra={"auto_default_from_env=True": True}
     )
 
     LOGIN_2FA_CODE_EXPIRATION_SEC: PositiveInt = Field(
@@ -94,7 +94,7 @@ def create_from_composition(
         """
         For the LoginSettings, product-specific settings override app-specifics settings
         """
-        composed_settings = {**app_login_settings.dict(), **product_login_settings}
+        composed_settings = {**app_login_settings.model_dump(), **product_login_settings}
 
         if "two_factor_enabled" in composed_settings:
             # legacy safe

services/web/server/src/simcore_service_webserver/products/_model.py

@@ -1,9 +1,9 @@
 import logging
 import string
-from typing import (  # noqa: UP035 # pydantic does not validate with re.Pattern
-    Any,
-    Pattern,
+from typing import (
+    Any
 )
+import re
 
 from models_library.basic_regex import (
     PUBLIC_VARIABLE_NAME_RE,
@@ -44,14 +44,13 @@ class Product(BaseModel):
     display_name: str = Field(..., description="Long display name")
     short_name: str | None = Field(
         None,
-        pattern=TWILIO_ALPHANUMERIC_SENDER_ID_RE,
+        pattern=re.compile(TWILIO_ALPHANUMERIC_SENDER_ID_RE),
         min_length=2,
         max_length=11,
         description="Short display name for SMS",
     )
 
-    host_regex: Pattern = Field(..., description="Host regex")
-    # NOTE: typing.Pattern is supported but not re.Pattern (SEE https://github.com/pydantic/pydantic/pull/4366)
+    host_regex: re.Pattern = Field(..., description="Host regex")
 
     support_email: LowerCaseEmailStr = Field(
         ...,
@@ -110,15 +109,15 @@ class Product(BaseModel):
 
     @field_validator("*", mode="before")
     @classmethod
-    def parse_empty_string_as_null(cls, v):
+    def _parse_empty_string_as_null(cls, v):
         """Safe measure: database entries are sometimes left blank instead of null"""
         if isinstance(v, str) and len(v.strip()) == 0:
             return None
         return v
 
     @field_validator("name", mode="before")
     @classmethod
-    def validate_name(cls, v):
+    def _validate_name(cls, v):
         if v not in FRONTEND_APPS_AVAILABLE:
             msg = f"{v} is not in available front-end apps {FRONTEND_APPS_AVAILABLE}"
             raise ValueError(msg)

services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py

@@ -4,11 +4,11 @@
 from pydantic import TypeAdapter, field_validator, ByteSize
 from pydantic.fields import Field
 from pydantic_settings import SettingsConfigDict
+
+from common_library.pydantic_networks_extension import HttpUrlLegacy
 from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY
 from settings_library.base import BaseCustomSettings
 
-from simcore_service_webserver.studies_dispatcher._rest_handlers import HttpUrlLegacy
-
 
 class StudiesDispatcherSettings(BaseCustomSettings):
     STUDIES_ACCESS_ANONYMOUS_ALLOWED: bool = Field(