🎨 Refactor API key pruning task: Simplify periodic execution and enhance logging

Author: pcrespov

services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_api_keys.py

@@ -6,71 +6,64 @@
 import asyncio
 import logging
 from collections.abc import AsyncIterator, Callable
+from datetime import timedelta
 
 from aiohttp import web
-from tenacity import retry
-from tenacity.before_sleep import before_sleep_log
-from tenacity.wait import wait_exponential
+from servicelib.async_utils import cancel_wait_task
+from servicelib.background_task_utils import exclusive_periodic
+from servicelib.logging_utils import log_context
+from simcore_service_webserver.redis import get_redis_lock_manager_client_sdk
 
 from ..api_keys import api_keys_service
 
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 CleanupContextFunc = Callable[[web.Application], AsyncIterator[None]]
 
 
-_PERIODIC_TASK_NAME = f"{__name__}.prune_expired_api_keys_periodically"
-_APP_TASK_KEY = f"{_PERIODIC_TASK_NAME}.task"
-
-
-@retry(
-    wait=wait_exponential(min=5, max=30),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-)
 async def _run_task(app: web.Application):
-    """Periodically check expiration dates and updates user status
-
-    It is resilient, i.e. if update goes wrong, it waits a bit and retries
-    """
+    """Checks expiration dates and updates user status"""
     if deleted := await api_keys_service.prune_expired_api_keys(app):
         # broadcast force logout of user_id
         for api_key in deleted:
-            logger.info("API-key %s expired and was removed", f"{api_key=}")
+            _logger.info("API-key %s expired and was removed", f"{api_key=}")
 
     else:
-        logger.info("No API keys expired")
+        _logger.info("No API keys expired")
 
 
-async def _run_periodically(app: web.Application, wait_period_s: float):
-    """Periodically check expiration dates and updates user status
+def create_background_task_to_prune_api_keys(
+    wait_period_s: float,
+) -> CleanupContextFunc:
 
-    It is resilient, i.e. if update goes wrong, it waits a bit and retries
-    """
-    while True:
-        await _run_task(app)
-        await asyncio.sleep(wait_period_s)
+    async def _cleanup_ctx_fun(app: web.Application) -> AsyncIterator[None]:
 
+        @exclusive_periodic(
+            # Function-exclusiveness is required to avoid multiple tasks like thisone running concurrently
+            get_redis_lock_manager_client_sdk(app),
+            task_interval=timedelta(seconds=wait_period_s),
+            retry_after=timedelta(minutes=5),
+        )
+        async def _prune_expired_api_keys_periodically() -> None:
+            with log_context(_logger, logging.INFO, "Pruning expired API keys"):
+                await _run_task(app)
 
-def create_background_task_to_prune_api_keys(
-    wait_period_s: float, task_name: str = _PERIODIC_TASK_NAME
-) -> CleanupContextFunc:
-    async def _cleanup_ctx_fun(
-        app: web.Application,
-    ) -> AsyncIterator[None]:
         # setup
+        task_name = _prune_expired_api_keys_periodically.__name__
+
         task = asyncio.create_task(
-            _run_periodically(app, wait_period_s),
+            _prune_expired_api_keys_periodically(),
             name=task_name,
         )
-        app[_APP_TASK_KEY] = task
+
+        # prevents premature garbage collection of the task
+        app_task_key = f"tasks.{task_name}"
+        app[app_task_key] = task
 
         yield
 
         # tear-down
-        task.cancel()
-        try:
-            await task
-        except asyncio.CancelledError:
-            assert task.cancelled()  # nosec
+        await cancel_wait_task(task)
+        app.pop(app_task_key, None)
 
     return _cleanup_ctx_fun