🎨 Persistent logs when GC removes services

packages/service-library/src/servicelib/logging_utils.py

@@ -392,3 +392,8 @@ def guess_message_log_level(message: str) -> LogLevelInt:
     ):
         return logging.WARNING
     return logging.INFO
+
+
+def set_parent_module_log_level(current_module: str, desired_log_level: int) -> None:
+    parent_module = ".".join(current_module.split(".")[:-1])
+    logging.getLogger(parent_module).setLevel(desired_log_level)

packages/service-library/tests/test_logging_utils.py

@@ -14,6 +14,7 @@
     log_context,
     log_decorator,
     log_exceptions,
+    set_parent_module_log_level,
 )
 
 _logger = logging.getLogger(__name__)
@@ -322,3 +323,57 @@ def test_log_exceptions_and_reraise(caplog: pytest.LogCaptureFixture, level: int
 
     assert len(caplog.records) == (1 if level != logging.NOTSET else 0)
     assert all(r.levelno == level for r in caplog.records)
+
+
+def test_set_parent_module_log_level_(caplog: pytest.LogCaptureFixture):
+    caplog.clear()
+    # emulates service logger
+    logging.root.setLevel(logging.WARNING)
+
+    parent = logging.getLogger("parent")
+    child = logging.getLogger("parent.child")
+
+    assert parent.level == logging.NOTSET
+    assert child.level == logging.NOTSET
+
+    parent.debug("parent debug")
+    child.debug("child debug")
+
+    parent.info("parent info")
+    child.info("child info")
+
+    parent.warning("parent warning")
+    child.warning("child warning")
+
+    assert "parent debug" not in caplog.text
+    assert "child debug" not in caplog.text
+
+    assert "parent info" not in caplog.text
+    assert "child info" not in caplog.text
+
+    assert "parent warning" in caplog.text
+    assert "child warning" in caplog.text
+
+    caplog.clear()
+    set_parent_module_log_level("parent.child", logging.INFO)
+
+    assert parent.level == logging.INFO
+    assert child.level == logging.NOTSET
+
+    parent.debug("parent debug")
+    child.debug("child debug")
+
+    parent.info("parent info")
+    child.info("child info")
+
+    parent.warning("parent warning")
+    child.warning("child warning")
+
+    assert "parent debug" not in caplog.text
+    assert "child debug" not in caplog.text
+
+    assert "parent info" in caplog.text
+    assert "child info" in caplog.text
+
+    assert "parent warning" in caplog.text
+    assert "child warning" in caplog.text

services/web/server/src/simcore_service_webserver/garbage_collector/_core_disconnected.py

@@ -106,6 +106,9 @@ async def remove_disconnected_user_resources(
                     # inform that the project can be closed on the backend side
                     #
                     try:
+                        _logger.info(
+                            "Closing services for project '%s'", resource_value
+                        )
                         await remove_project_dynamic_services(
                             user_id=user_id,
                             project_uuid=f"{resource_value}",

services/web/server/src/simcore_service_webserver/garbage_collector/plugin.py

@@ -2,7 +2,9 @@
 
 from aiohttp import web
 from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup
+from servicelib.logging_utils import set_parent_module_log_level
 
+from ..application_settings import get_application_settings
 from ..login.plugin import setup_login_storage
 from ..projects.db import setup_projects_db
 from ..socketio.plugin import setup_socketio
@@ -11,14 +13,14 @@
 from ._tasks_users import create_background_task_for_trial_accounts
 from .settings import get_plugin_settings
 
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 @app_module_setup(
     "simcore_service_webserver.garbage_collector",
     ModuleCategory.ADDON,
     settings_name="WEBSERVER_GARBAGE_COLLECTOR",
-    logger=logger,
+    logger=_logger,
 )
 def setup_garbage_collector(app: web.Application) -> None:
     # - project-api needs access to db
@@ -32,6 +34,10 @@ def setup_garbage_collector(app: web.Application) -> None:
 
     app.cleanup_ctx.append(run_background_task)
 
+    set_parent_module_log_level(
+        _logger.name, min(logging.INFO, get_application_settings(app).log_level)
+    )
+
     # NOTE: scaling web-servers will lead to having multiple tasks upgrading the db
     # not a huge deal. Instead this task runs in the GC.
     # If more tasks of this nature are needed, we should setup some sort of registration mechanism