tuning retry

Author: pcrespov

services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_api_keys.py

@@ -39,7 +39,7 @@ async def _cleanup_ctx_fun(app: web.Application) -> AsyncIterator[None]:
             # Function-exclusiveness is required to avoid multiple tasks like thisone running concurrently
             get_redis_lock_manager_client_sdk(app),
             task_interval=interval,
-            retry_after=interval,
+            retry_after=min(timedelta(seconds=10), interval / 10),
         )
         async def _prune_expired_api_keys_periodically() -> None:
             with log_context(_logger, logging.INFO, "Pruning expired API keys"):

services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_core.py

@@ -32,7 +32,7 @@ async def _cleanup_ctx_fun(app: web.Application) -> AsyncIterator[None]:
             # Function-exclusiveness is required to avoid multiple tasks like thisone running concurrently
             get_redis_lock_manager_client_sdk(app),
             task_interval=interval,
-            retry_after=interval,
+            retry_after=min(timedelta(seconds=10), interval / 10),
         )
         async def _collect_garbage_periodically() -> None:
             with log_context(_logger, logging.INFO, "Garbage collect cycle"):

services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_trash.py

@@ -27,7 +27,7 @@ async def _cleanup_ctx_fun(app: web.Application) -> AsyncIterator[None]:
             # Function-exclusiveness is required to avoid multiple tasks like thisone running concurrently
             get_redis_lock_manager_client_sdk(app),
             task_interval=interval,
-            retry_after=interval,
+            retry_after=min(timedelta(seconds=10), interval / 10),
         )
         async def _prune_trash_periodically() -> None:
             with log_context(_logger, logging.INFO, "Deleting expired trashed items"):

services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_users.py

@@ -76,7 +76,7 @@ async def _cleanup_ctx_fun(app: web.Application) -> AsyncIterator[None]:
             # Function-exclusiveness is required to avoid multiple tasks like thisone running concurrently
             get_redis_lock_manager_client_sdk(app),
             task_interval=interval,
-            retry_after=interval,
+            retry_after=min(timedelta(seconds=10), interval / 10),
         )
         async def _update_expired_users_periodically() -> None:
             with log_context(_logger, logging.INFO, "Updating expired users"):