introduce logger filtering

Author: matusdrobuliak66

.env-devel

@@ -124,6 +124,7 @@ INVITATIONS_SWAGGER_API_DOC_ENABLED=1
 INVITATIONS_USERNAME=admin
 
 LOG_FORMAT_LOCAL_DEV_ENABLED=1
+LOG_FILTER_MAPPING='{}'
 
 PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=30
 PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e6935107b57676

packages/service-library/src/servicelib/logging_utils.py

@@ -86,7 +86,9 @@ def format(self, record) -> str:
 # log_level=%{WORD:log_level} \| log_timestamp=%{TIMESTAMP_ISO8601:log_timestamp} \| log_source=%{DATA:log_source} \| log_msg=%{GREEDYDATA:log_msg}
 
 
-def config_all_loggers(*, log_format_local_dev_enabled: bool) -> None:
+def config_all_loggers(
+    *, log_format_local_dev_enabled: bool, logger_filter_mapping: dict[str, list[str]]
+) -> None:
     """
     Applies common configuration to ALL registered loggers
     """
@@ -106,18 +108,7 @@ def config_all_loggers(*, log_format_local_dev_enabled: bool) -> None:
             logger, fmt=fmt, log_format_local_dev_enabled=log_format_local_dev_enabled
         )
 
-    LOGGER_FILTER_MAPPING = {
-        "uvicorn.access": [
-            '"GET / HTTP/1.1" 200',
-            '"GET /metrics HTTP/1.1" 200',
-        ],
-        "gunicorn.access": [
-            '"GET /v0/ HTTP/1.1" 200',
-            '"GET /v0/health HTTP/1.1" 200',
-        ],
-    }
-    logger_mapping = LOGGER_FILTER_MAPPING
-    for logger_name, filtered_routes in logger_mapping.items():
+    for logger_name, filtered_routes in logger_filter_mapping.items():
         logger = logging.getLogger(logger_name)
         # Check if the logger has any handlers or is in active use
         if not logger.hasHandlers():

services/agent/src/simcore_service_agent/core/application.py

@@ -30,7 +30,8 @@ def _setup_logger(settings: ApplicationSettings):
     logging.basicConfig(level=settings.LOGLEVEL.value)  # NOSONAR
     logging.root.setLevel(settings.LOGLEVEL.value)
     config_all_loggers(
-        log_format_local_dev_enabled=settings.AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED
+        log_format_local_dev_enabled=settings.AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED,
+        logger_filter_mapping={},
     )
 
 

services/agent/src/simcore_service_agent/core/settings.py

@@ -25,6 +25,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
             "disabled if you want to have structured logs!"
         ),
     )
+    AGENT_VOLUMES_LOG_FILTER_MAPPING: dict = Field(
+        default={},
+        env=["AGENT_VOLUMES_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"],
+        description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.",
+    )
     AGENT_VOLUMES_CLEANUP_TARGET_SWARM_STACK_NAME: str = Field(
         ..., description="Exactly the same as director-v2's `SWARM_STACK_NAME` env var"
     )

services/api-server/src/simcore_service_api_server/core/application.py

@@ -52,7 +52,8 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI:
     logging.basicConfig(level=settings.log_level)
     logging.root.setLevel(settings.log_level)
     config_all_loggers(
-        log_format_local_dev_enabled=settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED
+        log_format_local_dev_enabled=settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED,
+        logger_filter_mapping={},
     )
     _logger.debug("App settings:\n%s", settings.json(indent=2))
 

services/api-server/src/simcore_service_api_server/core/settings.py

@@ -55,6 +55,11 @@ class BasicSettings(BaseCustomSettings, MixinLoggingSettings):
         env=["API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"],
         description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!",
     )
+    API_SERVER_LOG_FILTER_MAPPING: dict = Field(
+        default={},
+        env=["API_SERVER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"],
+        description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.",
+    )
 
     @validator("LOG_LEVEL", pre=True)
     @classmethod

services/autoscaling/src/simcore_service_autoscaling/core/settings.py

@@ -240,6 +240,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
         ],
         description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!",
     )
+    AUTOSCALING_LOG_FILTER_MAPPING: dict = Field(
+        default={},
+        env=["AUTOSCALING_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"],
+        description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.",
+    )
 
     AUTOSCALING_EC2_ACCESS: AutoscalingEC2Settings | None = Field(
         auto_default_from_env=True

services/autoscaling/src/simcore_service_autoscaling/main.py

@@ -13,7 +13,8 @@
 logging.basicConfig(level=the_settings.log_level)
 logging.root.setLevel(the_settings.log_level)
 config_all_loggers(
-    log_format_local_dev_enabled=the_settings.AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED
+    log_format_local_dev_enabled=the_settings.AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED,
+    logger_filter_mapping={},
 )
 
 # SINGLETON FastAPI app

services/catalog/src/simcore_service_catalog/core/settings.py

@@ -61,6 +61,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
         env=["CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"],
         description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!",
     )
+    CATALOG_LOG_FILTER_MAPPING: dict = Field(
+        default={},
+        env=["CATALOG_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"],
+        description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.",
+    )
     CATALOG_DEV_FEATURES_ENABLED: bool = Field(
         default=False,
         description="Enables development features. WARNING: make sure it is disabled in production .env file!",

services/catalog/src/simcore_service_catalog/main.py

@@ -14,7 +14,8 @@
 logging.basicConfig(level=_the_settings.CATALOG_LOG_LEVEL.value)  # NOSONAR
 logging.root.setLevel(_the_settings.CATALOG_LOG_LEVEL.value)
 config_all_loggers(
-    log_format_local_dev_enabled=_the_settings.CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED
+    log_format_local_dev_enabled=_the_settings.CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED,
+    logger_filter_mapping={},
 )