diff --git a/packages/common-library/src/common_library/async_tools.py b/packages/common-library/src/common_library/async_tools.py index 68c22212b14b..fc716303b02d 100644 --- a/packages/common-library/src/common_library/async_tools.py +++ b/packages/common-library/src/common_library/async_tools.py @@ -2,12 +2,15 @@ import datetime import functools import logging +import sys from collections.abc import Awaitable, Callable, Coroutine from concurrent.futures import Executor from functools import wraps from inspect import isawaitable from typing import Any, ParamSpec, TypeVar, overload +from .logging.logging_errors import create_troubleshooting_log_kwargs + _logger = logging.getLogger(__name__) R = TypeVar("R") @@ -102,11 +105,13 @@ async def cancel_wait_task( # from observing the cancellation/finalization of task. asyncio.wait_for(task, timeout=max_delay) ) - except TimeoutError: + except TimeoutError as exc: _logger.exception( - "Timeout while cancelling task %s after %s seconds", - task.get_name(), - max_delay, + **create_troubleshooting_log_kwargs( + f"Timeout while cancelling task {task.get_name()} after {max_delay} seconds", + error=exc, + error_context={"task_name": task.get_name(), "max_delay": max_delay}, + ) ) raise except asyncio.CancelledError: @@ -117,7 +122,20 @@ async def cancel_wait_task( raise finally: if not task.done(): - _logger.error("Failed to cancel %s", task.get_name()) + current_exception = sys.exception() + _logger.error( + **create_troubleshooting_log_kwargs( + f"Failed to cancel ask {task.get_name()}", + error=( + current_exception if current_exception else Exception("Unknown") + ), + error_context={ + "task_name": task.get_name(), + "max_delay": max_delay, + }, + tip="Consider increasing max_delay or fixing the task to handle cancellations properly", + ) + ) else: _logger.debug("Task %s cancelled", task.get_name()) diff --git a/packages/common-library/src/common_library/logging/__init__.py b/packages/common-library/src/common_library/logging/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/service-library/src/servicelib/logging_base.py b/packages/common-library/src/common_library/logging/logging_base.py similarity index 100% rename from packages/service-library/src/servicelib/logging_base.py rename to packages/common-library/src/common_library/logging/logging_base.py diff --git a/packages/service-library/src/servicelib/logging_errors.py b/packages/common-library/src/common_library/logging/logging_errors.py similarity index 95% rename from packages/service-library/src/servicelib/logging_errors.py rename to packages/common-library/src/common_library/logging/logging_errors.py index 0a251966a430..1c7c7d469b5c 100644 --- a/packages/service-library/src/servicelib/logging_errors.py +++ b/packages/common-library/src/common_library/logging/logging_errors.py @@ -1,5 +1,5 @@ import logging -from typing import Any, TypedDict +from typing import Any, Final, TypedDict from common_library.error_codes import ErrorCodeStr from common_library.errors_classes import OsparcErrorMixin @@ -9,6 +9,8 @@ _logger = logging.getLogger(__name__) +_MAX_LOGGED_CAUSES: Final[int] = 10 + def create_troubleshooting_log_message( user_error_msg: str, @@ -36,7 +38,7 @@ def _collect_causes(exc: BaseException) -> str: seen.add(id(current)) causes.append(f"[{type(current).__name__}]'{current}'") current = getattr(current, "__cause__", None) - if len(causes) > 10: # Prevent excessive chains + if len(causes) > _MAX_LOGGED_CAUSES: # Prevent excessive chains causes.append("[... truncated]") break return " <- ".join(causes) diff --git a/packages/service-library/src/servicelib/logging_utils_filtering.py b/packages/common-library/src/common_library/logging/logging_utils_filtering.py similarity index 100% rename from packages/service-library/src/servicelib/logging_utils_filtering.py rename to packages/common-library/src/common_library/logging/logging_utils_filtering.py diff --git a/packages/service-library/tests/test_logging_errors.py b/packages/common-library/tests/test_logging_errors.py similarity index 97% rename from packages/service-library/tests/test_logging_errors.py rename to packages/common-library/tests/test_logging_errors.py index 8bc1ef70aed8..481f152a9f08 100644 --- a/packages/service-library/tests/test_logging_errors.py +++ b/packages/common-library/tests/test_logging_errors.py @@ -5,7 +5,7 @@ import pytest from common_library.error_codes import create_error_code, parse_error_code_parts from common_library.errors_classes import OsparcErrorMixin -from servicelib.logging_errors import ( +from common_library.logging.logging_errors import ( create_troubleshooting_log_kwargs, create_troubleshooting_log_message, ) diff --git a/packages/service-library/tests/test_logging_utils_filtering.py b/packages/common-library/tests/test_logging_utils_filtering.py similarity index 96% rename from packages/service-library/tests/test_logging_utils_filtering.py rename to packages/common-library/tests/test_logging_utils_filtering.py index 64084c3204ac..75240cab747d 100644 --- a/packages/service-library/tests/test_logging_utils_filtering.py +++ b/packages/common-library/tests/test_logging_utils_filtering.py @@ -1,10 +1,10 @@ # pylint: disable=redefined-outer-name import logging -from typing import Generator +from collections.abc import Generator import pytest -from servicelib.logging_utils_filtering import GeneralLogFilter +from common_library.logging.logging_utils_filtering import GeneralLogFilter @pytest.fixture diff --git a/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py b/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py index fdb5df09d93c..7d6b99e22322 100644 --- a/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py +++ b/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py @@ -13,11 +13,11 @@ from aiohttp.web_response import StreamResponse from common_library.error_codes import ErrorCodeStr, create_error_code from common_library.json_serialization import json_dumps, json_loads +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from common_library.user_messages import user_message from models_library.basic_types import IDStr from models_library.rest_error import ErrorGet, ErrorItemType, LogMessageType -from ..logging_errors import create_troubleshooting_log_kwargs from ..mimetype_constants import MIMETYPE_APPLICATION_JSON from ..rest_constants import RESPONSE_MODEL_POLICY from ..rest_responses import is_enveloped_from_text @@ -202,7 +202,6 @@ def _handle_exception_as_http_error( def error_middleware_factory(api_version: str) -> Middleware: - @web.middleware async def _middleware_handler(request: web.Request, handler: Handler): """ diff --git a/packages/service-library/src/servicelib/exception_utils.py b/packages/service-library/src/servicelib/exception_utils.py index 7893763d13ba..76bfb149606f 100644 --- a/packages/service-library/src/servicelib/exception_utils.py +++ b/packages/service-library/src/servicelib/exception_utils.py @@ -5,10 +5,9 @@ from functools import wraps from typing import Any, Final, ParamSpec, TypeVar +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from pydantic import BaseModel, Field, NonNegativeFloat, PrivateAttr -from .logging_errors import create_troubleshooting_log_kwargs - _logger = logging.getLogger(__name__) _SKIPS_MESSAGE: Final[str] = "skip(s) of exception" diff --git a/packages/service-library/src/servicelib/fastapi/http_error.py b/packages/service-library/src/servicelib/fastapi/http_error.py index 9fbf2910811d..1b078111a7d9 100644 --- a/packages/service-library/src/servicelib/fastapi/http_error.py +++ b/packages/service-library/src/servicelib/fastapi/http_error.py @@ -2,6 +2,7 @@ from collections.abc import Awaitable, Callable from typing import TypeVar +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from fastapi import FastAPI, HTTPException, status from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError @@ -11,7 +12,6 @@ from fastapi.responses import JSONResponse from pydantic import ValidationError -from ..logging_errors import create_troubleshooting_log_kwargs from ..status_codes_utils import is_5xx_server_error validation_error_response_definition["properties"] = { diff --git a/packages/service-library/src/servicelib/fastapi/logging_lifespan.py b/packages/service-library/src/servicelib/fastapi/logging_lifespan.py index e06d976878b9..035d9bc10aa7 100644 --- a/packages/service-library/src/servicelib/fastapi/logging_lifespan.py +++ b/packages/service-library/src/servicelib/fastapi/logging_lifespan.py @@ -2,6 +2,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import AsyncExitStack +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from fastapi import FastAPI from settings_library.tracing import TracingSettings @@ -10,7 +11,6 @@ async_loggers, log_context, ) -from ..logging_utils_filtering import LoggerName, MessageSubstring from .lifespan_utils import Lifespan _logger = logging.getLogger(__name__) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py index 2af4ca5aa582..35e534fcc02a 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py @@ -5,9 +5,9 @@ from contextlib import asynccontextmanager from typing import Any, Final +from common_library.logging.logging_errors import create_troubleshooting_log_message from pydantic import PositiveFloat -from ...logging_errors import create_troubleshooting_log_message from ...long_running_tasks.errors import TaskClientTimeoutError, TaskExceptionError from ...long_running_tasks.models import ( ProgressCallback, diff --git a/packages/service-library/src/servicelib/logging_utils.py b/packages/service-library/src/servicelib/logging_utils.py index 613cf236a2ba..27bd08dd87f6 100644 --- a/packages/service-library/src/servicelib/logging_utils.py +++ b/packages/service-library/src/servicelib/logging_utils.py @@ -20,11 +20,15 @@ from typing import Any, Final, TypeAlias, TypedDict, TypeVar from common_library.json_serialization import json_dumps +from common_library.logging.logging_base import LogExtra +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs +from common_library.logging.logging_utils_filtering import ( + GeneralLogFilter, + LoggerName, + MessageSubstring, +) from settings_library.tracing import TracingSettings -from .logging_base import LogExtra -from .logging_errors import create_troubleshooting_log_kwargs -from .logging_utils_filtering import GeneralLogFilter, LoggerName, MessageSubstring from .tracing import setup_log_tracing from .utils_secrets import mask_sensitive_data @@ -74,9 +78,13 @@ def __init__(self, fmt: str, *, log_format_local_dev_enabled: bool) -> None: def format(self, record) -> str: if hasattr(record, "func_name_override"): - record.funcName = record.func_name_override + record.funcName = ( + record.func_name_override + ) # pyright: ignore[reportAttributeAccessIssue] if hasattr(record, "file_name_override"): - record.filename = record.file_name_override + record.filename = ( + record.file_name_override + ) # pyright: ignore[reportAttributeAccessIssue] # pylint: disable=no-member optional_keys = LogExtra.__optional_keys__ | frozenset( diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index d549f38f911e..adb76dc0699e 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -9,6 +9,7 @@ from uuid import uuid4 from common_library.async_tools import cancel_wait_task +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.api_schemas_long_running_tasks.base import TaskProgress from pydantic import NonNegativeFloat, PositiveFloat from settings_library.redis import RedisDatabase, RedisSettings @@ -20,7 +21,6 @@ ) from ..background_task import create_periodic_task -from ..logging_errors import create_troubleshooting_log_kwargs from ..logging_utils import log_catch, log_context from ..redis import RedisClientSDK, exclusive from ..utils import limited_gather diff --git a/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py b/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py index 6de24904577e..b7902bbd409e 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py +++ b/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py @@ -7,8 +7,8 @@ from typing import Any, TypeVar from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.rabbitmq_basic_types import RPCMethodName -from servicelib.logging_errors import create_troubleshooting_log_kwargs from ..logging_utils import log_context from ._errors import RPCServerError @@ -45,9 +45,7 @@ def expose( *, reraise_if_error_type: tuple[type[Exception], ...] | None = None, ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def _decorator(func: DecoratedCallable) -> DecoratedCallable: - @functools.wraps(func) async def _wrapper(*args, **kwargs): with log_context( diff --git a/packages/service-library/src/servicelib/redis/_decorators.py b/packages/service-library/src/servicelib/redis/_decorators.py index e1e724e05013..2ec079c5e7e6 100644 --- a/packages/service-library/src/servicelib/redis/_decorators.py +++ b/packages/service-library/src/servicelib/redis/_decorators.py @@ -9,10 +9,10 @@ import arrow import redis.exceptions from common_library.async_tools import cancel_wait_task +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from redis.asyncio.lock import Lock from ..background_task import periodic -from ..logging_errors import create_troubleshooting_log_kwargs from ._client import RedisClientSDK from ._constants import DEFAULT_LOCK_TTL from ._errors import CouldNotAcquireLockError, LockLostError diff --git a/packages/service-library/src/servicelib/redis/_semaphore_decorator.py b/packages/service-library/src/servicelib/redis/_semaphore_decorator.py index c2650251fe16..4a03e3928602 100644 --- a/packages/service-library/src/servicelib/redis/_semaphore_decorator.py +++ b/packages/service-library/src/servicelib/redis/_semaphore_decorator.py @@ -8,9 +8,9 @@ from typing import Any, ParamSpec, TypeVar from common_library.async_tools import cancel_wait_task +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from ..background_task import periodic -from ..logging_errors import create_troubleshooting_log_kwargs from ._client import RedisClientSDK from ._constants import ( DEFAULT_SEMAPHORE_TTL, diff --git a/packages/service-library/tests/test_logging_utils.py b/packages/service-library/tests/test_logging_utils.py index 39bb854b7dea..63f0d69c7883 100644 --- a/packages/service-library/tests/test_logging_utils.py +++ b/packages/service-library/tests/test_logging_utils.py @@ -10,8 +10,8 @@ from typing import Any import pytest +from common_library.logging.logging_base import get_log_record_extra from faker import Faker -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import ( _DEFAULT_FORMATTING, CustomFormatter, diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index d11b286f065f..edec12db2a57 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -2,10 +2,10 @@ from typing import Annotated from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from models_library.basic_types import BootModeEnum, LogLevel from models_library.docker import DockerNodeID from pydantic import AliasChoices, AnyHttpUrl, Field, field_validator -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.r_clone import S3Provider from settings_library.rabbit import RabbitSettings diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 031e1933777e..fe05d663d959 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -5,6 +5,7 @@ from celery_library.errors import TaskNotFoundError from common_library.exclude import as_dict_exclude_none +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.functions import ( FunctionClass, FunctionID, @@ -33,7 +34,6 @@ from models_library.users import UserID from servicelib.celery.models import TaskMetadata, TasksQueue, TaskUUID from servicelib.celery.task_manager import TaskManager -from servicelib.logging_errors import create_troubleshooting_log_kwargs from simcore_service_api_server.models.schemas.functions import ( FunctionJobCreationTaskStatus, ) @@ -137,15 +137,16 @@ async def list_function_jobs_with_status( pagination_offset=pagination_offset, pagination_limit=pagination_limit ) - function_jobs_list_ws, meta = ( - await self._web_rpc_client.list_function_jobs_with_status( - user_id=self.user_id, - product_name=self.product_name, - filter_by_function_id=filter_by_function_id, - filter_by_function_job_ids=filter_by_function_job_ids, - filter_by_function_job_collection_id=filter_by_function_job_collection_id, - **pagination_kwargs, - ) + ( + function_jobs_list_ws, + meta, + ) = await self._web_rpc_client.list_function_jobs_with_status( + user_id=self.user_id, + product_name=self.product_name, + filter_by_function_id=filter_by_function_id, + filter_by_function_job_ids=filter_by_function_job_ids, + filter_by_function_job_collection_id=filter_by_function_job_collection_id, + **pagination_kwargs, ) for function_job_wso in function_jobs_list_ws: @@ -297,7 +298,6 @@ async def function_job_outputs( function_job: RegisteredFunctionJob, stored_job_outputs: FunctionOutputs | None, ) -> FunctionOutputs: - if stored_job_outputs is not None: return stored_job_outputs @@ -360,7 +360,6 @@ async def create_function_job_creation_task( parent_project_uuid: ProjectID | None = None, parent_node_id: NodeID | None = None, ) -> RegisteredFunctionJob: - job_inputs = await self._function_job_service.create_function_job_inputs( function=function, function_inputs=function_inputs ) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/tasks.py b/services/api-server/src/simcore_service_api_server/api/routes/tasks.py index f29b30a585ef..42199f8d4308 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/tasks.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/tasks.py @@ -2,6 +2,7 @@ from typing import Annotated, Any from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from fastapi import APIRouter, Depends, FastAPI, HTTPException, status from models_library.api_schemas_long_running_tasks.base import TaskProgress from models_library.api_schemas_long_running_tasks.tasks import ( @@ -16,7 +17,6 @@ from models_library.users import UserID from servicelib.celery.models import TaskState, TaskUUID from servicelib.fastapi.dependencies import get_app -from servicelib.logging_errors import create_troubleshooting_log_kwargs from simcore_service_api_server.models.domain.celery_models import ApiWorkerTaskFilter from ...models.schemas.base import ApiServerEnvelope @@ -57,7 +57,6 @@ async def list_tasks( user_id: Annotated[UserID, Depends(get_current_user_id)], product_name: Annotated[ProductName, Depends(get_product_name)], ): - task_manager = get_task_manager(app) task_filter = ApiWorkerTaskFilter( user_id=user_id, diff --git a/services/api-server/src/simcore_service_api_server/core/settings.py b/services/api-server/src/simcore_service_api_server/core/settings.py index 9b622a5ddc9c..d4fbdfe89098 100644 --- a/services/api-server/src/simcore_service_api_server/core/settings.py +++ b/services/api-server/src/simcore_service_api_server/core/settings.py @@ -2,6 +2,7 @@ from typing import Annotated from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from models_library.basic_types import BootModeEnum, LogLevel from pydantic import ( AliasChoices, @@ -11,7 +12,6 @@ SecretStr, field_validator, ) -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.celery import CelerySettings from settings_library.director_v2 import DirectorV2Settings @@ -28,7 +28,6 @@ class WebServerSettings(WebServerBaseSettings, MixinSessionSettings): - WEBSERVER_SESSION_SECRET_KEY: Annotated[ SecretStr, Field( diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_backend_errors.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_backend_errors.py index dccfe3ff114a..619d52c122ad 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_backend_errors.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_backend_errors.py @@ -1,7 +1,7 @@ import logging from common_library.error_codes import create_error_code -from servicelib.logging_errors import create_troubleshooting_log_kwargs +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from servicelib.status_codes_utils import is_5xx_server_error from starlette.requests import Request from starlette.responses import JSONResponse diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_factory.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_factory.py index dc5e7b618e29..0af52ea3671c 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_factory.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_factory.py @@ -1,9 +1,9 @@ import logging from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from fastapi.requests import Request from fastapi.responses import JSONResponse -from servicelib.logging_errors import create_troubleshooting_log_kwargs from ._utils import ExceptionHandler, create_error_json_response diff --git a/services/api-server/src/simcore_service_api_server/services_http/log_streaming.py b/services/api-server/src/simcore_service_api_server/services_http/log_streaming.py index b30eb5fc5125..7d925aed9971 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/log_streaming.py +++ b/services/api-server/src/simcore_service_api_server/services_http/log_streaming.py @@ -5,10 +5,10 @@ from typing import Final from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.rabbitmq_messages import LoggerRabbitMessage from models_library.users import UserID from pydantic import NonNegativeInt -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_catch from servicelib.rabbitmq import QueueName, RabbitMQClient diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py index c18bf519362b..c240dcc68c32 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py @@ -3,6 +3,7 @@ from typing import Annotated, Final, Self, cast from aws_library.ec2 import EC2InstanceBootSpecific, EC2Tags +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from fastapi import FastAPI from models_library.basic_types import LogLevel, PortInt, VersionTag from models_library.clusters import ClusterAuthentication @@ -18,7 +19,6 @@ ) from pydantic_settings import SettingsConfigDict from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings diff --git a/services/catalog/src/simcore_service_catalog/core/settings.py b/services/catalog/src/simcore_service_catalog/core/settings.py index 5581bf4ba995..4237ce0c812c 100644 --- a/services/catalog/src/simcore_service_catalog/core/settings.py +++ b/services/catalog/src/simcore_service_catalog/core/settings.py @@ -3,6 +3,7 @@ from typing import Annotated, Final from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from models_library.api_schemas_catalog.services_specifications import ( ServiceSpecifications, ) @@ -16,7 +17,6 @@ PositiveInt, TypeAdapter, ) -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.base import BaseCustomSettings from settings_library.http_client_request import ClientRequestSettings diff --git a/services/catalog/src/simcore_service_catalog/service/catalog_services.py b/services/catalog/src/simcore_service_catalog/service/catalog_services.py index f4f0573440ee..f97b6da792d6 100644 --- a/services/catalog/src/simcore_service_catalog/service/catalog_services.py +++ b/services/catalog/src/simcore_service_catalog/service/catalog_services.py @@ -4,6 +4,7 @@ from contextlib import suppress from typing import Literal, TypeVar +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.api_schemas_catalog.services import ( LatestServiceGet, MyServiceGet, @@ -22,9 +23,6 @@ from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID from pydantic import HttpUrl -from servicelib.logging_errors import ( - create_troubleshooting_log_kwargs, -) from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( CatalogForbiddenError, CatalogInconsistentError, @@ -108,7 +106,6 @@ def _to_latest_get_schema( access_rights_db: list[ServiceAccessRightsDB], service_manifest: ServiceMetaDataPublished, ) -> LatestServiceGet: - assert len(service_db.history) == 0 # nosec return LatestServiceGet.model_validate( @@ -397,7 +394,6 @@ async def get_catalog_service( service_key: ServiceKey, service_version: ServiceVersion, ) -> ServiceGetV2: - access_rights = await check_catalog_service_permissions( repo=repo, product_name=product_name, @@ -449,7 +445,6 @@ async def update_catalog_service( service_version: ServiceVersion, update: ServiceUpdateV2, ) -> ServiceGetV2: - if is_function_service(service_key): raise CatalogForbiddenError( name=f"function service {service_key}:{service_version}", @@ -482,7 +477,6 @@ async def update_catalog_service( # Updates service_access_rights (they can be added/removed/modified) if update.access_rights: - # before previous_gids = [r.gid for r in access_rights] @@ -602,7 +596,6 @@ async def batch_get_user_services( ] ], ) -> list[MyServiceGet]: - services_access_rights = await repo.batch_get_services_access_rights( key_versions=ids, product_name=product_name ) @@ -612,7 +605,6 @@ async def batch_get_user_services( my_services = [] for service_key, service_version in ids: - # Evaluate user's access-rights to this service key:version access_rights = services_access_rights.get((service_key, service_version), []) my_access_rights = ServiceGroupAccessRightsV2(execute=False, write=False) @@ -695,7 +687,6 @@ async def list_user_service_release_history( # result options include_compatibility: bool = False, ) -> tuple[PageTotalCount, list[ServiceRelease]]: - total_count, history = await repo.get_service_history_page( # NOTE: that the service history might be different for each user # since access rights are defined on a version basis (i.e. one use can have access to v1 but ot to v2) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index aef3bd798e3a..d7f47dcf16e5 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -4,6 +4,7 @@ from aws_library.ec2 import EC2InstanceBootSpecific, EC2Tags from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from fastapi import FastAPI from models_library.basic_types import ( BootModeEnum, @@ -24,7 +25,6 @@ ) from pydantic_settings import SettingsConfigDict from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings from settings_library.ec2 import EC2Settings diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py index f26297681753..ae5b7d504064 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py @@ -2,10 +2,10 @@ from pathlib import Path from typing import Annotated, Any, cast +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from models_library.basic_types import LogLevel from pydantic import AliasChoices, Field, field_validator from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.rabbit import RabbitSettings from settings_library.utils_logging import MixinLoggingSettings diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py index 98f091c76e93..0b5089375123 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py @@ -1,10 +1,10 @@ from typing import Annotated from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from models_library.basic_types import LogLevel from pydantic import AliasChoices, Field, TypeAdapter, field_validator from pydantic.networks import AnyUrl -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.base import BaseCustomSettings from settings_library.tracing import TracingSettings diff --git a/services/director-v2/src/simcore_service_director_v2/api/errors/http_error.py b/services/director-v2/src/simcore_service_director_v2/api/errors/http_error.py index 8f630b7bfac1..ddf050d91a93 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/errors/http_error.py +++ b/services/director-v2/src/simcore_service_director_v2/api/errors/http_error.py @@ -1,9 +1,9 @@ import logging from collections.abc import Awaitable, Callable +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from fastapi import HTTPException from fastapi.encoders import jsonable_encoder -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.status_codes_utils import is_5xx_server_error from starlette.requests import Request from starlette.responses import JSONResponse diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index 26a7e0d3a514..0a85dc4cb313 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -7,6 +7,7 @@ from typing import Annotated, cast from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from common_library.pydantic_validators import validate_numeric_string_as_timedelta from fastapi import FastAPI from models_library.basic_types import LogLevel, PortInt @@ -24,7 +25,6 @@ PositiveInt, field_validator, ) -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.base import BaseCustomSettings from settings_library.catalog import CatalogSettings diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_manager.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_manager.py index 430dc0f94b2b..418d3fcde149 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_manager.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_manager.py @@ -185,8 +185,12 @@ async def schedule_all_pipelines(app: FastAPI) -> None: ) if possibly_lost_scheduled_pipelines: _logger.error( - "found %d lost pipelines, they will be re-scheduled now", + "found %d lost pipelines, they will be re-scheduled now. '%s'", len(possibly_lost_scheduled_pipelines), + ", ".join( + f"{run.project_uuid=}|{run.iteration=}" + for run in possibly_lost_scheduled_pipelines + ), ) rabbitmq_client = get_rabbitmq_client(app) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py index da882c263499..1491d2ef781c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py @@ -21,6 +21,7 @@ import arrow import networkx as nx +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from common_library.user_messages import user_message from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, NodeIDStr @@ -31,7 +32,6 @@ from networkx.classes.reportviews import InDegreeView from pydantic import PositiveInt from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_catch, log_context from servicelib.rabbitmq import RabbitMQClient, RabbitMQRPCClient from servicelib.redis import RedisClientSDK @@ -46,7 +46,6 @@ DaskClientAcquisisitonError, InvalidPipelineError, PipelineNotFoundError, - TaskSchedulingError, ) from ...core.settings import ComputationalBackendSettings from ...models.comp_pipelines import CompPipelineAtDB @@ -834,6 +833,13 @@ async def _schedule_tasks_to_start( # nothing to do return comp_tasks + log_error_context = { + "user_id": f"{user_id}", + "project_id": f"{project_id}", + "tasks_ready_to_start": f"{list(tasks_ready_to_start.keys())}", + "comp_run_use_on_demand_clusters": f"{comp_run.use_on_demand_clusters}", + "comp_run_run_id": f"{comp_run.run_id}", + } try: await self._start_tasks( user_id=user_id, @@ -842,35 +848,20 @@ async def _schedule_tasks_to_start( comp_run=comp_run, wake_up_callback=wake_up_callback, ) - except ( - ComputationalBackendNotConnectedError, - ComputationalSchedulerChangedError, - ClustersKeeperNotAvailableError, - ) as exc: - _logger.exception( + except ComputationalBackendOnDemandNotReadyError as exc: + _logger.info( **create_troubleshooting_log_kwargs( - "Computational backend is not connected. Tasks are set back " - "to WAITING_FOR_CLUSTER state until scheduler comes back!", + "The on demand computational backend is not ready yet. Tasks are set to WAITING_FOR_CLUSTER state until the cluster is ready!", error=exc, - error_context={ - "user_id": f"{user_id}", - "project_id": f"{project_id}", - "tasks_ready_to_start": f"{list(tasks_ready_to_start.keys())}", - "comp_run_use_on_demand_clusters": f"{comp_run.use_on_demand_clusters}", - "comp_run.run_id": f"{comp_run.run_id}", - }, + error_context=log_error_context, ) ) - await publish_project_log( self.rabbitmq_client, user_id, project_id, - log=user_message( - "An unexpected error occurred during task scheduling. Please contact oSparc support if this issue persists.", - _version=1, - ), - log_level=logging.ERROR, + log=f"{exc}", + log_level=logging.INFO, ) await CompTasksRepository.instance( self.db_engine @@ -882,29 +873,29 @@ async def _schedule_tasks_to_start( ) for task in tasks_ready_to_start: comp_tasks[f"{task}"].state = RunningState.WAITING_FOR_CLUSTER - - except ComputationalBackendOnDemandNotReadyError as exc: - _logger.info( + except ( + ComputationalBackendNotConnectedError, + ComputationalSchedulerChangedError, + ClustersKeeperNotAvailableError, + ) as exc: + _logger.exception( **create_troubleshooting_log_kwargs( - "The on demand computational backend is not ready yet. Tasks are set to WAITING_FOR_CLUSTER state until the cluster is ready!", + "Computational backend is not connected. Tasks are set back " + "to WAITING_FOR_CLUSTER state until scheduler comes back!", error=exc, - error_context={ - "user_id": f"{user_id}", - "project_id": f"{project_id}", - "tasks_ready_to_start": f"{list(tasks_ready_to_start.keys())}", - "comp_run_use_on_demand_clusters": f"{comp_run.use_on_demand_clusters}", - "comp_run.run_id": f"{comp_run.run_id}", - }, + error_context=log_error_context, ) ) await publish_project_log( self.rabbitmq_client, user_id, project_id, - log=f"{exc}", - log_level=logging.INFO, + log=user_message( + "An unexpected error occurred during task scheduling. Please contact oSparc support if this issue persists.", + _version=1, + ), + log_level=logging.ERROR, ) - await CompTasksRepository.instance( self.db_engine ).update_project_tasks_state( @@ -915,45 +906,13 @@ async def _schedule_tasks_to_start( ) for task in tasks_ready_to_start: comp_tasks[f"{task}"].state = RunningState.WAITING_FOR_CLUSTER - except TaskSchedulingError as exc: - _logger.exception( - **create_troubleshooting_log_kwargs( - "A task could not be scheduled, it is set to FAILED and the rest of the pipeline will be ABORTED", - error=exc, - error_context={ - "user_id": f"{user_id}", - "project_id": f"{project_id}", - "node_id": f"{exc.node_id}", - "tasks_ready_to_start": f"{list(tasks_ready_to_start.keys())}", - "comp_run_use_on_demand_clusters": f"{comp_run.use_on_demand_clusters}", - "comp_run.run_id": f"{comp_run.run_id}", - }, - ) - ) - await CompTasksRepository.instance( - self.db_engine - ).update_project_tasks_state( - project_id, - comp_run.run_id, - [exc.node_id], - RunningState.FAILED, - exc.get_errors(), - optional_progress=1.0, - optional_stopped=arrow.utcnow().datetime, - ) - comp_tasks[f"{exc.node_id}"].state = RunningState.FAILED + except Exception as exc: _logger.exception( **create_troubleshooting_log_kwargs( "Unexpected error happened when scheduling tasks, all tasks to start are set to FAILED and the rest of the pipeline will be ABORTED", error=exc, - error_context={ - "user_id": f"{comp_run.user_id}", - "project_id": f"{comp_run.project_uuid}", - "tasks_ready_to_start": f"{list(tasks_ready_to_start.keys())}", - "comp_run_use_on_demand_clusters": f"{comp_run.use_on_demand_clusters}", - "comp_run.run_id": f"{comp_run.run_id}", - }, + error_context=log_error_context, ) ) await CompTasksRepository.instance( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py index 04bdb0e799cf..3a4ae4a6f97f 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py @@ -6,6 +6,7 @@ from typing import Any, Final import arrow +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from dask_task_models_library.container_tasks.errors import TaskCancelledError from dask_task_models_library.container_tasks.events import ( TaskProgressEvent, @@ -22,7 +23,6 @@ from models_library.users import UserID from pydantic import PositiveInt from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_catch, log_context from servicelib.redis._client import RedisClientSDK from servicelib.redis._semaphore_decorator import ( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index 571d4acdaa08..735286b77a2d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -18,6 +18,7 @@ import distributed from aiohttp import ClientResponseError from common_library.json_serialization import json_dumps +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from dask_task_models_library.container_tasks.docker import DockerBasicAuth from dask_task_models_library.container_tasks.errors import TaskCancelledError from dask_task_models_library.container_tasks.events import TaskProgressEvent @@ -56,7 +57,6 @@ from models_library.users import UserID from pydantic import ValidationError from pydantic.networks import AnyUrl -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_context from servicelib.utils import limited_gather from settings_library.s3 import S3Settings diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py index 3ac029d4e426..3ebe33ced687 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py @@ -5,8 +5,8 @@ from math import floor from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from fastapi import FastAPI -from servicelib.logging_errors import create_troubleshooting_log_kwargs from .....core.dynamic_services_settings.scheduler import ( DynamicServicesSchedulerSettings, diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py index 134d03f05eca..8e3351317756 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py @@ -347,9 +347,14 @@ async def test_schedule_all_pipelines_logs_error_if_it_find_old_pipelines( ) with caplog.at_level(logging.ERROR): await schedule_all_pipelines(initialized_app) + lost_pipeline_messages = [ + msg + for msg in caplog.messages + if "lost pipelines" in msg and "re-scheduled" in msg + ] assert ( - "found 1 lost pipelines, they will be re-scheduled now" in caplog.messages - ) + len(lost_pipeline_messages) > 0 + ), f"Expected lost pipeline message, got: {caplog.messages}" _assert_scheduler_client_called_once_with( scheduler_rabbit_client_parser, SchedulePipelineRabbitMessage( diff --git a/services/director/src/simcore_service_director/core/settings.py b/services/director/src/simcore_service_director/core/settings.py index 2cfea3139183..7e1b010aae89 100644 --- a/services/director/src/simcore_service_director/core/settings.py +++ b/services/director/src/simcore_service_director/core/settings.py @@ -3,11 +3,11 @@ from functools import cached_property from typing import cast +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from fastapi import FastAPI from models_library.basic_types import LogLevel, PortInt, VersionTag from pydantic import AliasChoices, Field, NonNegativeInt, PositiveInt, field_validator from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.docker_registry import RegistrySettings from settings_library.postgres import PostgresSettings diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 8405a419536e..4083ef85f332 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -3,9 +3,9 @@ from typing import Annotated, cast from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from pydantic import AliasChoices, Field, SecretStr, TypeAdapter, field_validator from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag from settings_library.catalog import CatalogSettings diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index ffb972d5fcca..cf516f97770b 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import cast +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import PortInt from models_library.callbacks_mapping import CallbacksMapping @@ -21,7 +22,6 @@ TypeAdapter, field_validator, ) -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.aws_s3_cli import AwsS3CliSettings from settings_library.docker_registry import RegistrySettings @@ -55,7 +55,6 @@ class SystemMonitorSettings(BaseApplicationSettings): class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): - DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR: Path = Field( ..., description="Base directory where dynamic-sidecar stores creates " diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py index 1634f0b1b2c0..239d9b5122ce 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py @@ -8,7 +8,7 @@ import psutil from common_library.error_codes import create_error_code -from servicelib.logging_errors import create_troubleshooting_log_kwargs +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from ..modules.mounted_fs import MountedVolumes @@ -106,7 +106,6 @@ async def async_command( ) except Exception as err: # pylint: disable=broad-except - error_code = create_error_code(err) user_error_msg = f"Unexpected error [{error_code}]" _logger.exception( diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index cda71422f9ed..7894e4343a4c 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -3,11 +3,11 @@ from typing import Annotated, Final, cast from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from fastapi import FastAPI from models_library.basic_types import LogLevel, VersionTag from pydantic import AliasChoices, ByteSize, Field, TypeAdapter, field_validator from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.efs import AwsEfsSettings from settings_library.postgres import PostgresSettings diff --git a/services/invitations/src/simcore_service_invitations/core/exceptions_handlers.py b/services/invitations/src/simcore_service_invitations/core/exceptions_handlers.py index c4209b542292..ba46db5b96cd 100644 --- a/services/invitations/src/simcore_service_invitations/core/exceptions_handlers.py +++ b/services/invitations/src/simcore_service_invitations/core/exceptions_handlers.py @@ -1,8 +1,8 @@ import logging +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from fastapi import FastAPI, Request, status from fastapi.responses import JSONResponse -from servicelib.logging_errors import create_troubleshooting_log_kwargs from ..services.invitations import InvalidInvitationCodeError diff --git a/services/invitations/src/simcore_service_invitations/core/settings.py b/services/invitations/src/simcore_service_invitations/core/settings.py index 44ab579cbc72..ba4141cb7a7b 100644 --- a/services/invitations/src/simcore_service_invitations/core/settings.py +++ b/services/invitations/src/simcore_service_invitations/core/settings.py @@ -2,10 +2,10 @@ from typing import Annotated, cast from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from models_library.products import ProductName from pydantic import AliasChoices, Field, HttpUrl, SecretStr, field_validator from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag from settings_library.tracing import TracingSettings diff --git a/services/notifications/src/simcore_service_notifications/core/settings.py b/services/notifications/src/simcore_service_notifications/core/settings.py index 6f7e13a546e3..89d66154b042 100644 --- a/services/notifications/src/simcore_service_notifications/core/settings.py +++ b/services/notifications/src/simcore_service_notifications/core/settings.py @@ -1,9 +1,9 @@ from typing import Annotated from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from models_library.basic_types import BootModeEnum, LogLevel from pydantic import AliasChoices, Field, field_validator -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.postgres import PostgresSettings from settings_library.rabbit import RabbitSettings diff --git a/services/payments/src/simcore_service_payments/api/rest/_acknowledgements.py b/services/payments/src/simcore_service_payments/api/rest/_acknowledgements.py index 73b28c165918..a76fef903f72 100644 --- a/services/payments/src/simcore_service_payments/api/rest/_acknowledgements.py +++ b/services/payments/src/simcore_service_payments/api/rest/_acknowledgements.py @@ -1,12 +1,12 @@ import logging from typing import Annotated +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status from models_library.api_schemas_payments.errors import ( PaymentMethodNotFoundError, PaymentNotFoundError, ) -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_context from ..._constants import ACKED, PGDB diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments.py b/services/payments/src/simcore_service_payments/api/rpc/_payments.py index 66420c509887..801b47e512ec 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments.py @@ -1,6 +1,7 @@ import logging from decimal import Decimal +from common_library.logging.logging_base import get_log_record_extra from fastapi import FastAPI from models_library.api_schemas_payments.errors import ( PaymentsError, @@ -16,7 +17,6 @@ from models_library.users import UserID from models_library.wallets import WalletID from pydantic import EmailStr, HttpUrl -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from servicelib.rabbitmq import RPCRouter diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py index 8d7f208c1c61..8c93fd7bde6e 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py @@ -1,6 +1,7 @@ import logging from decimal import Decimal +from common_library.logging.logging_base import get_log_record_extra from fastapi import FastAPI from models_library.api_schemas_payments.errors import ( PaymentsError, @@ -18,7 +19,6 @@ from models_library.users import UserID from models_library.wallets import WalletID from pydantic import EmailStr -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from servicelib.rabbitmq import RPCRouter diff --git a/services/payments/src/simcore_service_payments/core/settings.py b/services/payments/src/simcore_service_payments/core/settings.py index 67932ac466db..8d018a6b4414 100644 --- a/services/payments/src/simcore_service_payments/core/settings.py +++ b/services/payments/src/simcore_service_payments/core/settings.py @@ -3,6 +3,7 @@ from typing import Annotated, cast from common_library.basic_types import DEFAULT_FACTORY +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from models_library.basic_types import NonNegativeDecimal from pydantic import ( AliasChoices, @@ -15,7 +16,6 @@ field_validator, ) from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag from settings_library.email import SMTPSettings diff --git a/services/payments/src/simcore_service_payments/services/notifier_email.py b/services/payments/src/simcore_service_payments/services/notifier_email.py index 2d9b222e785e..8c6f41452dc6 100644 --- a/services/payments/src/simcore_service_payments/services/notifier_email.py +++ b/services/payments/src/simcore_service_payments/services/notifier_email.py @@ -10,12 +10,12 @@ import httpx from aiosmtplib import SMTP from attr import dataclass +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from jinja2 import DictLoader, Environment, select_autoescape from models_library.api_schemas_webserver.wallets import PaymentMethodTransaction from models_library.products import ProductName from models_library.users import UserID from pydantic import EmailStr -from servicelib.logging_errors import create_troubleshooting_log_kwargs from settings_library.email import EmailProtocol, SMTPSettings from tenacity import ( retry, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index 3590c3847b93..18f5dd6795a5 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -2,10 +2,10 @@ from functools import cached_property from typing import cast +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from models_library.basic_types import BootModeEnum from pydantic import AliasChoices, Field, PositiveInt, field_validator from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.basic_types import BuildTargetEnum, LogLevel, VersionTag from settings_library.postgres import PostgresSettings diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py index f09fc5e69842..d660259577e8 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py @@ -1,9 +1,9 @@ import logging from collections.abc import Awaitable, Callable +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from fastapi import HTTPException, status from fastapi.encoders import jsonable_encoder -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.status_codes_utils import is_5xx_server_error from starlette.requests import Request from starlette.responses import JSONResponse diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index 9d49f4660ba8..a15061179b5a 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -1,8 +1,8 @@ from typing import Annotated, Self +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from fastapi import FastAPI from pydantic import AliasChoices, Field, PositiveInt, field_validator, model_validator -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, PortInt from settings_library.celery import CelerySettings diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index b9ce6ace3004..d99623469475 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -5,6 +5,7 @@ from aiohttp import web from common_library.basic_types import DEFAULT_FACTORY from common_library.exclude import Unset +from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from common_library.pydantic_fields_extension import is_nullable from models_library.basic_types import LogLevel, PortInt, VersionTag from models_library.utils.change_case import snake_to_camel @@ -17,7 +18,6 @@ ) from pydantic.fields import Field from servicelib.logging_utils import LogLevelInt -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.email import SMTPSettings from settings_library.postgres import PostgresSettings @@ -612,7 +612,6 @@ def to_client_statics(self) -> dict[str, Any]: def setup_settings(app: web.Application) -> ApplicationSettings: - settings: ApplicationSettings = ApplicationSettings.create_from_envs() app[APP_SETTINGS_KEY] = settings _logger.debug( diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py index 839042f2a01c..b3728e979658 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py @@ -4,10 +4,10 @@ from aiohttp import web from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from common_library.user_messages import user_message from models_library.rest_error import ErrorGet from servicelib.aiohttp import status -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.rabbitmq._errors import RemoteMethodNotRegisteredError from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( CatalogForbiddenError, @@ -42,7 +42,6 @@ async def _handler_catalog_client_errors( request: web.Request, exception: Exception ) -> web.Response: - assert isinstance( # nosec exception, CatalogResponseError | CatalogConnectionError ), f"check mapping, got {exception=}" diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_controller/_rest_exceptions.py b/services/web/server/src/simcore_service_webserver/director_v2/_controller/_rest_exceptions.py index 6376f622fe9f..89888cb31757 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_controller/_rest_exceptions.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_controller/_rest_exceptions.py @@ -2,11 +2,11 @@ from aiohttp import web from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from common_library.user_messages import user_message from models_library.rest_error import ErrorGet from servicelib import status_codes_utils from servicelib.aiohttp import status -from servicelib.logging_errors import create_troubleshooting_log_kwargs from ...constants import MSG_TRY_AGAIN_OR_SUPPORT from ...exception_handling import ( diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py index b7fe746ccfa2..0cd58f93a8f0 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py @@ -2,6 +2,7 @@ from uuid import UUID from aiohttp import web +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.api_schemas_directorv2.computations import ( TasksOutputs, TasksSelection, @@ -16,7 +17,6 @@ from pydantic.types import PositiveInt from servicelib.aiohttp import status from servicelib.exception_utils import suppress_exceptions -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_decorator from simcore_postgres_database.utils_groups_extra_properties import ( GroupExtraProperties, diff --git a/services/web/server/src/simcore_service_webserver/email/_handlers.py b/services/web/server/src/simcore_service_webserver/email/_handlers.py index 9e96a388205a..8ba75dee3b7b 100644 --- a/services/web/server/src/simcore_service_webserver/email/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/email/_handlers.py @@ -2,10 +2,10 @@ from typing import Any, Literal from aiohttp import web +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.emails import LowerCaseEmailStr from pydantic import BaseModel, Field from servicelib.aiohttp.requests_validation import parse_request_body_as -from servicelib.logging_errors import create_troubleshooting_log_kwargs from .._meta import API_VTAG from ..login.decorators import login_required @@ -91,7 +91,6 @@ async def test_email(request: web.Request): ) except Exception as err: # pylint: disable=broad-except - _logger.exception( **create_troubleshooting_log_kwargs( user_error_msg="Email test failed", diff --git a/services/web/server/src/simcore_service_webserver/exception_handling/_factory.py b/services/web/server/src/simcore_service_webserver/exception_handling/_factory.py index 7f8f42251cc1..49118ad8b8de 100644 --- a/services/web/server/src/simcore_service_webserver/exception_handling/_factory.py +++ b/services/web/server/src/simcore_service_webserver/exception_handling/_factory.py @@ -4,10 +4,10 @@ from aiohttp import web from common_library.error_codes import create_error_code from common_library.json_serialization import json_dumps +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.rest_error import ErrorGet from servicelib.aiohttp.rest_responses import safe_status_message from servicelib.aiohttp.web_exceptions_extension import get_all_aiohttp_http_exceptions -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.status_codes_utils import ( get_code_display_name, is_5xx_server_error, @@ -88,7 +88,6 @@ async def _exception_handler( request: web.Request, exception: BaseException, ) -> web.Response: - # safe formatting, i.e. does not raise user_msg = msg_template.format_map( _DefaultDict(getattr(exception, "__dict__", {})) diff --git a/services/web/server/src/simcore_service_webserver/garbage_collector/_core_orphans.py b/services/web/server/src/simcore_service_webserver/garbage_collector/_core_orphans.py index e1f176c89dfe..31d79dced2ed 100644 --- a/services/web/server/src/simcore_service_webserver/garbage_collector/_core_orphans.py +++ b/services/web/server/src/simcore_service_webserver/garbage_collector/_core_orphans.py @@ -2,6 +2,7 @@ from typing import Final from aiohttp import web +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from common_library.users_enums import UserRole from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( @@ -9,7 +10,6 @@ ) from models_library.projects_nodes_io import NodeID from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_catch, log_context from servicelib.utils import limited_as_completed, limited_gather @@ -54,10 +54,13 @@ async def _remove_service( ) ) - with log_catch(_logger, reraise=False), log_context( - _logger, - logging.INFO, - f"removing {(service.node_uuid, service.host)} with {save_service_state=}", + with ( + log_catch(_logger, reraise=False), + log_context( + _logger, + logging.INFO, + f"removing {(service.node_uuid, service.host)} with {save_service_state=}", + ), ): await dynamic_scheduler_service.stop_dynamic_service( app, diff --git a/services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_users.py b/services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_users.py index efda89ee747f..25caef6bbd92 100644 --- a/services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_users.py +++ b/services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_users.py @@ -8,9 +8,9 @@ from datetime import timedelta from aiohttp import web +from common_library.logging.logging_base import get_log_record_extra from models_library.users import UserID from servicelib.background_task_utils import exclusive_periodic -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from ..login import login_service diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py index 0e8d4ba3bbcc..1e1155613e25 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py @@ -14,6 +14,7 @@ import sqlalchemy as sa from aiohttp import web from aiopg.sa.result import RowProxy +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from pydantic import ( BaseModel, Field, @@ -23,7 +24,6 @@ ValidationError, field_validator, ) -from servicelib.logging_errors import create_troubleshooting_log_kwargs from simcore_postgres_database.models.classifiers import group_classifiers from ..db.plugin import get_database_engine_legacy diff --git a/services/web/server/src/simcore_service_webserver/login/_controller/rest/auth.py b/services/web/server/src/simcore_service_webserver/login/_controller/rest/auth.py index de6ca2dc1068..3b260c7008b4 100644 --- a/services/web/server/src/simcore_service_webserver/login/_controller/rest/auth.py +++ b/services/web/server/src/simcore_service_webserver/login/_controller/rest/auth.py @@ -2,12 +2,12 @@ from aiohttp import web from aiohttp.web import RouteTableDef +from common_library.logging.logging_base import get_log_record_extra from common_library.user_messages import user_message from models_library.authentification import TwoFactorAuthentificationMethod from pydantic import TypeAdapter from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_body_as -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from servicelib.request_keys import RQT_USERID_KEY from simcore_postgres_database.models.users import UserRole diff --git a/services/web/server/src/simcore_service_webserver/login/_controller/rest/change.py b/services/web/server/src/simcore_service_webserver/login/_controller/rest/change.py index b87934e389eb..7abc08c8ffb9 100644 --- a/services/web/server/src/simcore_service_webserver/login/_controller/rest/change.py +++ b/services/web/server/src/simcore_service_webserver/login/_controller/rest/change.py @@ -2,8 +2,8 @@ from aiohttp import web from aiohttp.web import RouteTableDef +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from servicelib.aiohttp.requests_validation import parse_request_body_as -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.request_keys import RQT_USERID_KEY from simcore_postgres_database.utils_users import UsersRepo @@ -270,7 +270,6 @@ async def initiate_change_email(request: web.Request): @routes.post(f"/{API_VTAG}/auth/change-password", name="auth_change_password") @login_required async def change_password(request: web.Request): - passwords = await parse_request_body_as(ChangePasswordBody, request) user_id = request[RQT_USERID_KEY] user = await _auth_service.get_user_or_none(request.app, user_id=user_id) diff --git a/services/web/server/src/simcore_service_webserver/login/_controller/rest/confirmation.py b/services/web/server/src/simcore_service_webserver/login/_controller/rest/confirmation.py index e62a2ae4c13d..7003eef09a0d 100644 --- a/services/web/server/src/simcore_service_webserver/login/_controller/rest/confirmation.py +++ b/services/web/server/src/simcore_service_webserver/login/_controller/rest/confirmation.py @@ -3,6 +3,7 @@ from aiohttp import web from aiohttp.web import RouteTableDef from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName from pydantic import ( @@ -13,7 +14,6 @@ parse_request_body_as, parse_request_path_parameters_as, ) -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from yarl import URL diff --git a/services/web/server/src/simcore_service_webserver/login/_controller/rest/registration.py b/services/web/server/src/simcore_service_webserver/login/_controller/rest/registration.py index a4a636d87de8..081bbe650416 100644 --- a/services/web/server/src/simcore_service_webserver/login/_controller/rest/registration.py +++ b/services/web/server/src/simcore_service_webserver/login/_controller/rest/registration.py @@ -4,9 +4,9 @@ from aiohttp import web from aiohttp.web import RouteTableDef from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_body_as -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from simcore_postgres_database.models.users import UserStatus diff --git a/services/web/server/src/simcore_service_webserver/login/_invitations_service.py b/services/web/server/src/simcore_service_webserver/login/_invitations_service.py index 3cd8bac78174..7d753fb26713 100644 --- a/services/web/server/src/simcore_service_webserver/login/_invitations_service.py +++ b/services/web/server/src/simcore_service_webserver/login/_invitations_service.py @@ -11,6 +11,7 @@ from aiohttp import web from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.basic_types import IdInt from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName @@ -23,7 +24,6 @@ ValidationError, field_validator, ) -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from simcore_postgres_database.models.confirmations import ConfirmationAction from simcore_postgres_database.models.users import UserStatus @@ -115,12 +115,10 @@ async def check_other_registrations( db: AsyncpgStorage, cfg: LoginOptions, ) -> None: - # An account is already registered with this email if user := await _auth_service.get_user_or_none(app, email=email): user_status = UserStatus(user["status"]) match user_status: - case UserStatus.ACTIVE: await _raise_if_registered_in_product( app, user_email=user["email"], product=current_product diff --git a/services/web/server/src/simcore_service_webserver/login/_security_service.py b/services/web/server/src/simcore_service_webserver/login/_security_service.py index 14b263088599..932c02287193 100644 --- a/services/web/server/src/simcore_service_webserver/login/_security_service.py +++ b/services/web/server/src/simcore_service_webserver/login/_security_service.py @@ -3,7 +3,7 @@ import logging from aiohttp import web -from servicelib.logging_base import get_log_record_extra +from common_library.logging.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from ..security import security_web diff --git a/services/web/server/src/simcore_service_webserver/login_accounts/_controller_rest.py b/services/web/server/src/simcore_service_webserver/login_accounts/_controller_rest.py index 04ff7151b9cb..80f37c8b362e 100644 --- a/services/web/server/src/simcore_service_webserver/login_accounts/_controller_rest.py +++ b/services/web/server/src/simcore_service_webserver/login_accounts/_controller_rest.py @@ -2,6 +2,7 @@ from typing import Any from aiohttp import web +from common_library.logging.logging_base import get_log_record_extra from common_library.user_messages import user_message from models_library.api_schemas_webserver.auth import ( AccountRequestInfo, @@ -13,7 +14,6 @@ handle_validation_as_http_error, parse_request_body_as, ) -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from servicelib.utils import fire_and_forget_task diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py index 9835b97f3549..925ca516f2c0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py @@ -4,6 +4,7 @@ from aiohttp import web from common_library.error_codes import create_error_code from common_library.json_serialization import json_dumps +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from common_library.user_messages import user_message from models_library.api_schemas_catalog.service_access_rights import ( ServiceAccessRightsGet, @@ -47,7 +48,6 @@ UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, X_SIMCORE_USER_AGENT, ) -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.long_running_tasks.models import TaskProgress from servicelib.long_running_tasks.task import TaskRegistry from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON diff --git a/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py b/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py index f00d99bf8cbc..a6a8ab7216be 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py @@ -8,6 +8,7 @@ from typing import cast from aiohttp import web +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.api_schemas_webserver.projects import ( ProjectDocument, ProjectDocumentVersion, @@ -15,7 +16,6 @@ from models_library.api_schemas_webserver.socketio import SocketIORoomStr from models_library.projects import ProjectID, ProjectTemplateType from models_library.projects import ProjectType as ProjectTypeAPI -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_context from servicelib.redis import ( PROJECT_DB_UPDATE_REDIS_LOCK_KEY, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 11bf8dbe9f11..bbe8d8e35215 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -15,6 +15,7 @@ from aiopg.sa import Engine from aiopg.sa.connection import SAConnection from aiopg.sa.result import ResultProxy, RowProxy +from common_library.logging.logging_base import get_log_record_extra from models_library.basic_types import IDStr from models_library.folders import FolderQuery, FolderScope from models_library.groups import GroupID @@ -41,7 +42,6 @@ from pydantic import TypeAdapter from pydantic.types import PositiveInt from servicelib.aiohttp.application_keys import APP_AIOPG_ENGINE_KEY -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from simcore_postgres_database.aiopg_errors import UniqueViolation from simcore_postgres_database.models.groups import user_to_groups diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 01b442ff8159..d5ec51cbea0b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -22,6 +22,7 @@ from aiohttp import web from common_library.json_serialization import json_dumps +from common_library.logging.logging_base import get_log_record_extra from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.api_schemas_directorv2.dynamic_services import ( DynamicServiceGet, @@ -84,7 +85,6 @@ X_FORWARDED_PROTO, X_SIMCORE_USER_AGENT, ) -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from servicelib.rabbitmq import RemoteMethodNotRegisteredError, RPCServerError from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/user_sessions.py b/services/web/server/src/simcore_service_webserver/resource_manager/user_sessions.py index 41370df9edcd..15253d645731 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/user_sessions.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/user_sessions.py @@ -6,8 +6,8 @@ from typing import Final from aiohttp import web +from common_library.logging.logging_base import get_log_record_extra from models_library.users import UserID -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from .models import ResourcesDict, UserSession diff --git a/services/web/server/src/simcore_service_webserver/security/_authz_policy.py b/services/web/server/src/simcore_service_webserver/security/_authz_policy.py index 08a594e257a1..20f5705a6bd0 100644 --- a/services/web/server/src/simcore_service_webserver/security/_authz_policy.py +++ b/services/web/server/src/simcore_service_webserver/security/_authz_policy.py @@ -10,11 +10,11 @@ from aiohttp_security.abc import ( # type: ignore[import-untyped] AbstractAuthorizationPolicy, ) +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from common_library.users_enums import UserRole from models_library.products import ProductName from models_library.users import UserID from servicelib.aiohttp.db_asyncpg_engine import get_async_engine -from servicelib.logging_errors import create_troubleshooting_log_kwargs from simcore_postgres_database.aiopg_errors import DatabaseError as AiopgDatabaseError from sqlalchemy.exc import DatabaseError as SQLAlchemyDatabaseError diff --git a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py index 5e11a422a476..95e718ae45f2 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py @@ -9,14 +9,14 @@ import socketio.exceptions # type: ignore[import-untyped] from aiohttp import web +from common_library.logging.logging_base import get_log_record_extra +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.api_schemas_webserver.socketio import SocketIORoomStr from models_library.products import ProductName from models_library.socketio import SocketMessageDict from models_library.users import UserID from pydantic import TypeAdapter from servicelib.aiohttp.observer import emit -from servicelib.logging_base import get_log_record_extra -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_context from servicelib.request_keys import RQT_USERID_KEY diff --git a/services/web/server/src/simcore_service_webserver/socketio/_observer.py b/services/web/server/src/simcore_service_webserver/socketio/_observer.py index 16e991abc6b3..292e24c32fd0 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/_observer.py +++ b/services/web/server/src/simcore_service_webserver/socketio/_observer.py @@ -7,9 +7,9 @@ import logging from aiohttp import web +from common_library.logging.logging_base import get_log_record_extra from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from servicelib.aiohttp.observer import register_observer, setup_observer_registry -from servicelib.logging_base import get_log_record_extra from servicelib.utils import fire_and_forget_task, logged_gather from socketio import AsyncServer # type: ignore[import-untyped] diff --git a/services/web/server/src/simcore_service_webserver/storage/api.py b/services/web/server/src/simcore_service_webserver/storage/api.py index 300f88441490..d68f8bd94eea 100644 --- a/services/web/server/src/simcore_service_webserver/storage/api.py +++ b/services/web/server/src/simcore_service_webserver/storage/api.py @@ -7,6 +7,7 @@ from typing import Any, Final from aiohttp import ClientError, ClientSession, ClientTimeout, web +from common_library.logging.logging_base import get_log_record_extra from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE from models_library.api_schemas_storage.storage_schemas import ( FileLocation, @@ -22,7 +23,6 @@ from models_library.users import UserID from pydantic import ByteSize, HttpUrl, TypeAdapter from servicelib.aiohttp.client_session import get_client_session -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from servicelib.rabbitmq.rpc_interfaces.async_jobs.async_jobs import ( AsyncJobComposedResult, diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_exceptions.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_exceptions.py index e5110c8730d5..094a7cebac99 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_exceptions.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_exceptions.py @@ -3,11 +3,11 @@ from aiohttp import web from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from common_library.user_messages import user_message from models_library.function_services_catalog._utils import ServiceNotFound from servicelib.aiohttp import status from servicelib.aiohttp.typing_extension import Handler -from servicelib.logging_errors import create_troubleshooting_log_kwargs from ....exception_handling import create_error_context_from_request from ....utils import compose_support_error_msg diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py index 3c25e491cfa6..77a83efe4071 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py @@ -21,10 +21,10 @@ from aiohttp import web from aiohttp_session import get_session from common_library.error_codes import create_error_code +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.projects import ProjectID from servicelib.aiohttp import status from servicelib.aiohttp.typing_extension import Handler -from servicelib.logging_errors import create_troubleshooting_log_kwargs from ..constants import INDEX_RESOURCE_NAME from ..director_v2 import director_v2_service diff --git a/services/web/server/src/simcore_service_webserver/trash/_service.py b/services/web/server/src/simcore_service_webserver/trash/_service.py index d436a1eedb85..055598866790 100644 --- a/services/web/server/src/simcore_service_webserver/trash/_service.py +++ b/services/web/server/src/simcore_service_webserver/trash/_service.py @@ -5,9 +5,9 @@ import arrow from aiohttp import web +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.products import ProductName from models_library.users import UserID -from servicelib.logging_errors import create_troubleshooting_log_kwargs from servicelib.logging_utils import log_context from ..folders import folders_trash_service diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 69d95703bdf0..8d8ab85e299c 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -2,6 +2,7 @@ import logging from aiohttp import web +from common_library.logging.logging_base import get_log_record_extra from models_library.api_schemas_webserver.wallets import ( CreateWalletPayment, GetWalletAutoRecharge, @@ -21,7 +22,6 @@ parse_request_path_parameters_as, parse_request_query_parameters_as, ) -from servicelib.logging_base import get_log_record_extra from servicelib.logging_utils import log_context from servicelib.utils import fire_and_forget_task