30817 - AUTH-API - Retry keycloak add/remove groups on failure (bcgov#3581)

Author: Jxio

auth-api/poetry.lock

@@ -56,6 +56,7 @@ sbc-common-components = { git = "https://github.com/bcgov/sbc-common-components.
 cloud-sql-connector = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/cloud-sql-connector", branch = "main" }
 gcp-queue = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/gcp-queue", branch = "main" }
 structured-logging = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/structured-logging", branch = "main" }
+aiohttp-retry = "^2.9.1"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^8.3.2"

auth-api/pyproject.toml

@@ -20,6 +20,7 @@
 
 import aiohttp
 import requests
+from aiohttp_retry import ExponentialRetry, RetryClient
 from flask import current_app
 
 from auth_api.exceptions import BusinessException
@@ -286,15 +287,19 @@ async def add_or_remove_users_from_group(kgs: list[KeycloakGroupSubscription]):
         method = "PUT" if kgs[0].group_action == KeycloakGroupActions.ADD_TO_GROUP.value else "DELETE"
         # Normal limit is 100, cap this to 40, so it doesn't hit keycloak too aggressively.
         connector = aiohttp.TCPConnector(limit=40)
-        async with aiohttp.ClientSession(connector=connector) as session:
+        retry_options = ExponentialRetry(
+            attempts=3,
+            start_timeout=1,
+            statuses={500, 502, 503, 504},
+            exceptions={TimeoutError, aiohttp.ClientConnectionError}
+        )
+        async with RetryClient(connector=connector, retry_options=retry_options) as session:
             tasks = [
-                asyncio.create_task(
-                    session.request(
-                        method,
-                        f"{base_url}/auth/admin/realms/{realm}/users/{kg.user_guid}/groups/{group_ids[kg.group_name]}",
-                        headers=headers,
-                        timeout=timeout,
-                    )
+                session.request(
+                    method,
+                    f"{base_url}/auth/admin/realms/{realm}/users/{kg.user_guid}/groups/{group_ids[kg.group_name]}",
+                    headers=headers,
+                    timeout=timeout
                 )
                 for kg in kgs
             ]