Merge remote-tracking branch 'upstream/pydantic_v2_migration_do_not_squash_updates' into pr-osparc-pydantic-v2-fixes-2

Author: Andrei Neagu

.env-devel

@@ -10,7 +10,7 @@
 #     unset  $(grep -v '^#' .env | sed -E 's/(.*)=.*/\1/' | xargs)
 #
 
-AGENT_LOGLEVEL=WARNING
+AGENT_LOGLEVEL=INFO
 AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY=12345678
 AGENT_VOLUMES_CLEANUP_S3_BUCKET=simcore-volume-backups
 AGENT_VOLUMES_CLEANUP_S3_ENDPOINT=http://172.17.0.1:9001
@@ -19,7 +19,7 @@ AGENT_VOLUMES_CLEANUP_S3_REGION=us-east-1
 AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY=12345678
 
 API_SERVER_DEV_FEATURES_ENABLED=0
-API_SERVER_LOGLEVEL=WARNING
+API_SERVER_LOGLEVEL=INFO
 API_SERVER_PROFILING=1
 TRAEFIK_API_SERVER_INFLIGHTREQ_AMOUNT=25
 
@@ -29,7 +29,7 @@ AUTOSCALING_DOCKER_JOIN_DRAINED=True
 AUTOSCALING_WAIT_FOR_CLOUD_INIT_BEFORE_WARM_BUFFER_ACTIVATION=False
 AUTOSCALING_EC2_ACCESS=null
 AUTOSCALING_EC2_INSTANCES=null
-AUTOSCALING_LOGLEVEL=WARNING
+AUTOSCALING_LOGLEVEL=INFO
 AUTOSCALING_NODES_MONITORING=null
 AUTOSCALING_POLL_INTERVAL="00:00:10"
 AUTOSCALING_SSM_ACCESS=null
@@ -39,7 +39,7 @@ AWS_S3_CLI_S3=null
 CATALOG_BACKGROUND_TASK_REST_TIME=60
 CATALOG_DEV_FEATURES_ENABLED=0
 CATALOG_HOST=catalog
-CATALOG_LOGLEVEL=WARNING
+CATALOG_LOGLEVEL=INFO
 CATALOG_PORT=8000
 CATALOG_PROFILING=1
 CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": {"limit": 2147483648, "reservation": 2147483648}}'
@@ -52,15 +52,15 @@ CLUSTERS_KEEPER_DASK_WORKER_SATURATION=inf
 CLUSTERS_KEEPER_EC2_ACCESS=null
 CLUSTERS_KEEPER_SSM_ACCESS=null
 CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX=""
-CLUSTERS_KEEPER_LOGLEVEL=WARNING
+CLUSTERS_KEEPER_LOGLEVEL=INFO
 CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=5
 CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null
 CLUSTERS_KEEPER_TASK_INTERVAL=30
 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null
 
 DASK_SCHEDULER_HOST=dask-scheduler
 DASK_SCHEDULER_PORT=8786
-DASK_SIDECAR_LOGLEVEL=WARNING
+DASK_SIDECAR_LOGLEVEL=INFO
 DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem
 DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem
 DASK_TLS_KEY=/home/scu/.dask/dask-key.pem
@@ -91,7 +91,7 @@ DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REA
 DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL=0
 DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 DIRECTOR_V2_HOST=director-v2
-DIRECTOR_V2_LOGLEVEL=WARNING
+DIRECTOR_V2_LOGLEVEL=INFO
 DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH=null
 DIRECTOR_V2_PORT=8000
 DIRECTOR_V2_PROFILING=1
@@ -115,7 +115,7 @@ FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]
 # Can use 'docker run -it itisfoundation/invitations:latest simcore-service-invitations generate-dotenv --auto-password'
 INVITATIONS_DEFAULT_PRODUCT=osparc
 INVITATIONS_HOST=invitations
-INVITATIONS_LOGLEVEL=WARNING
+INVITATIONS_LOGLEVEL=INFO
 INVITATIONS_OSPARC_URL=http://127.0.0.1.nip.io:9081
 INVITATIONS_PASSWORD=adminadmin
 INVITATIONS_PORT=8000
@@ -124,6 +124,7 @@ INVITATIONS_SWAGGER_API_DOC_ENABLED=1
 INVITATIONS_USERNAME=admin
 
 LOG_FORMAT_LOCAL_DEV_ENABLED=1
+LOG_FILTER_MAPPING='{}'
 
 PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=30
 PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e6935107b57676
@@ -138,7 +139,7 @@ PAYMENTS_FAKE_COMPLETION=0
 PAYMENTS_GATEWAY_API_SECRET=adminadmin
 PAYMENTS_GATEWAY_URL=http://127.0.0.1:32769
 PAYMENTS_HOST=payments
-PAYMENTS_LOGLEVEL=WARNING
+PAYMENTS_LOGLEVEL=INFO
 PAYMENTS_PASSWORD=adminadmin
 PAYMENTS_PORT=8000
 PAYMENTS_STRIPE_API_SECRET='REPLACE_ME_with_api_secret'
@@ -179,7 +180,7 @@ RESOURCE_MANAGER_RESOURCE_TTL_S=900
 RESOURCE_USAGE_TRACKER_HOST=resource-usage-tracker
 RESOURCE_USAGE_TRACKER_PORT=8000
 RESOURCE_USAGE_TRACKER_EXTERNAL_PORT=8000
-RESOURCE_USAGE_TRACKER_LOGLEVEL=WARNING
+RESOURCE_USAGE_TRACKER_LOGLEVEL=INFO
 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED=1
 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL=6
 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC=300
@@ -214,7 +215,7 @@ BF_API_KEY=none
 BF_API_SECRET=none
 STORAGE_ENDPOINT=storage:8080
 STORAGE_HOST=storage
-STORAGE_LOGLEVEL=WARNING
+STORAGE_LOGLEVEL=INFO
 STORAGE_PORT=8080
 STORAGE_PROFILING=1
 # STORAGE ----
@@ -245,7 +246,7 @@ WB_GC_GARBAGE_COLLECTOR='{"GARBAGE_COLLECTOR_INTERVAL_S": 30}'
 WB_GC_GROUPS=0
 WB_GC_INVITATIONS=null
 WB_GC_LOGIN=null
-WB_GC_LOGLEVEL=WARNING
+WB_GC_LOGLEVEL=INFO
 WB_GC_META_MODELING=0
 WB_GC_NOTIFICATIONS=0
 WB_GC_PAYMENTS=null
@@ -278,7 +279,7 @@ WB_DB_EL_GARBAGE_COLLECTOR=null
 WB_DB_EL_GROUPS=0
 WB_DB_EL_INVITATIONS=null
 WB_DB_EL_LOGIN=null
-WB_DB_EL_LOGLEVEL=WARNING
+WB_DB_EL_LOGLEVEL=INFO
 WB_DB_EL_META_MODELING=0
 WB_DB_EL_NOTIFICATIONS=0
 WB_DB_EL_PAYMENTS=null
@@ -348,7 +349,7 @@ WEBSERVER_GROUPS=1
 WEBSERVER_GUNICORN_CMD_ARGS=--timeout=180
 WEBSERVER_HOST=webserver
 WEBSERVER_LOGIN={}
-WEBSERVER_LOGLEVEL=WARNING
+WEBSERVER_LOGLEVEL=INFO
 WEBSERVER_META_MODELING=1
 WEBSERVER_NOTIFICATIONS=1
 WEBSERVER_PAYMENTS={}

packages/aws-library/tests/test_s3_client.py

@@ -696,7 +696,7 @@ async def test_create_single_presigned_download_link(
 
     dest_file = tmp_path / faker.file_name()
     async with ClientSession() as session:
-        response = await session.get(download_url)
+        response = await session.get(f"{download_url}")
         response.raise_for_status()
         with dest_file.open("wb") as fp:
             fp.write(await response.read())

packages/models-library/src/models_library/docker.py

@@ -75,7 +75,7 @@ def to_simcore_runtime_docker_label_key(key: str) -> DockerLabelKey:
 class StandardSimcoreDockerLabels(BaseModel):
     """
     Represents the standard label on oSparc created containers (not yet services)
-    In order to create this object in code, please use construct() method!
+    In order to create this object in code, please use model_construct() method!
     """
 
     user_id: UserID = Field(..., alias=f"{_SIMCORE_RUNTIME_DOCKER_LABEL_PREFIX}user-id")  # type: ignore[literal-required]

packages/postgres-database/tests/test_utils_projects.py

@@ -12,7 +12,7 @@
 from aiopg.sa.connection import SAConnection
 from aiopg.sa.result import RowProxy
 from faker import Faker
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 from simcore_postgres_database.models.projects import projects
 from simcore_postgres_database.utils_projects import (
     DBProjectNotFoundError,
@@ -69,7 +69,7 @@ async def test_get_project_trashed_at_column_can_be_converted_to_datetime(
 
         row = result.fetchone()
 
-    trashed_at = parse_obj_as(datetime | None, row.trashed_at)
+    trashed_at = TypeAdapter(datetime | None).validate_python(row.trashed_at)
     assert trashed_at == expected
 
 

packages/service-library/src/servicelib/logging_utils.py

@@ -16,6 +16,7 @@
 from pathlib import Path
 from typing import Any, NotRequired, TypeAlias, TypedDict, TypeVar
 
+from .logging_utils_filtering import GeneralLogFilter, LoggerName, MessageSubstring
 from .utils_secrets import mask_sensitive_data
 
 _logger = logging.getLogger(__name__)
@@ -86,7 +87,11 @@ def format(self, record) -> str:
 # log_level=%{WORD:log_level} \| log_timestamp=%{TIMESTAMP_ISO8601:log_timestamp} \| log_source=%{DATA:log_source} \| log_msg=%{GREEDYDATA:log_msg}
 
 
-def config_all_loggers(*, log_format_local_dev_enabled: bool) -> None:
+def config_all_loggers(
+    *,
+    log_format_local_dev_enabled: bool,
+    logger_filter_mapping: dict[LoggerName, list[MessageSubstring]],
+) -> None:
     """
     Applies common configuration to ALL registered loggers
     """
@@ -102,12 +107,25 @@ def config_all_loggers(*, log_format_local_dev_enabled: bool) -> None:
         fmt = LOCAL_FORMATTING
 
     for logger in loggers:
-        set_logging_handler(
+        _set_logging_handler(
             logger, fmt=fmt, log_format_local_dev_enabled=log_format_local_dev_enabled
         )
 
+    for logger_name, filtered_routes in logger_filter_mapping.items():
+        logger = logging.getLogger(logger_name)
+        # Check if the logger has any handlers or is in active use
+        if not logger.hasHandlers():
+            _logger.warning(
+                "Logger %s does not have any handlers. Filter will not be added.",
+                logger_name,
+            )
+            continue
+
+        log_filter = GeneralLogFilter(filtered_routes)
+        logger.addFilter(log_filter)
+
 
-def set_logging_handler(
+def _set_logging_handler(
     logger: logging.Logger,
     *,
     fmt: str,

packages/service-library/src/servicelib/logging_utils_filtering.py

@@ -0,0 +1,28 @@
+"""
+This codes originates from this article
+    https://medium.com/swlh/add-log-decorators-to-your-python-project-84094f832181
+
+SEE also https://github.com/Delgan/loguru for a future alternative
+"""
+
+import logging
+from typing import TypeAlias
+
+_logger = logging.getLogger(__name__)
+
+LoggerName: TypeAlias = str
+MessageSubstring: TypeAlias = str
+
+
+class GeneralLogFilter(logging.Filter):
+    def __init__(self, filtered_routes: list[str]) -> None:
+        super().__init__()
+        self.filtered_routes = filtered_routes
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        msg = record.getMessage()
+
+        # Check if the filtered routes exists in the message
+        return not any(
+            filter_criteria in msg for filter_criteria in self.filtered_routes
+        )

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/volumes.py

@@ -4,7 +4,7 @@
 
 from models_library.projects_nodes_io import NodeID
 from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace
-from pydantic import NonNegativeInt, parse_obj_as
+from pydantic import NonNegativeInt, TypeAdapter
 from servicelib.logging_utils import log_decorator
 from servicelib.rabbitmq import RabbitMQRPCClient
 
@@ -29,7 +29,9 @@ async def remove_volumes_without_backup_for_service(
                 "swarm_stack_name": swarm_stack_name,
             }
         ),
-        parse_obj_as(RPCMethodName, "remove_volumes_without_backup_for_service"),
+        TypeAdapter(RPCMethodName).validate_python(
+            "remove_volumes_without_backup_for_service"
+        ),
         node_id=node_id,
         timeout_s=_REQUEST_TIMEOUT,
     )
@@ -51,7 +53,9 @@ async def backup_and_remove_volumes_for_all_services(
                 "swarm_stack_name": swarm_stack_name,
             }
         ),
-        parse_obj_as(RPCMethodName, "backup_and_remove_volumes_for_all_services"),
+        TypeAdapter(RPCMethodName).validate_python(
+            "backup_and_remove_volumes_for_all_services"
+        ),
         timeout_s=_REQUEST_TIMEOUT,
     )
     assert result is None  # nosec

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py

@@ -1,14 +1,20 @@
 import logging
+from typing import Final
 
 from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage
 from models_library.projects_nodes_io import NodeID
 from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace
-from pydantic import parse_obj_as
-from servicelib.logging_utils import log_decorator
-from servicelib.rabbitmq import RabbitMQRPCClient
+from pydantic import TypeAdapter
+
+from ....logging_utils import log_decorator
+from ... import RabbitMQRPCClient
 
 _logger = logging.getLogger(__name__)
 
+_UPDATE_DISK_USAGE: Final[RPCMethodName] = TypeAdapter(RPCMethodName).validate_python(
+    "update_disk_usage"
+)
+
 
 @log_decorator(_logger, level=logging.DEBUG)
 async def update_disk_usage(
@@ -21,6 +27,8 @@ async def update_disk_usage(
         {"service": "dy-sidecar", "node_id": f"{node_id}"}
     )
     result = await rabbitmq_rpc_client.request(
-        rpc_namespace, parse_obj_as(RPCMethodName, "update_disk_usage"), usage=usage
+        rpc_namespace,
+        _UPDATE_DISK_USAGE,
+        usage=usage,
     )
     assert result is None  # nosec

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Final, cast
+from typing import Final
 
 from models_library.api_schemas_resource_usage_tracker import (
     RESOURCE_USAGE_TRACKER_RPC_NAMESPACE,
@@ -115,5 +115,5 @@ async def export_service_runs(
         filters=filters,
         timeout_s=_DEFAULT_TIMEOUT_S,
     )
-    assert cast(AnyUrl, isinstance(result, AnyUrl))  # nosec
+    assert isinstance(result, AnyUrl)  # nosec
     return result