Fix/sentry connection closed error (#53)

* Adding retry logic

* updated retry logic

* added slack alert logic

* added module: slack_sdk

Author: feba-rajan

request-api/requirements/requirements.txt

@@ -136,6 +136,7 @@ shortuuid==1.0.13
     # via -r requirements/requirements.in
 six==1.16.0
     # via python-dateutil
+slack_sdk>=3.0.0
 sniffio==1.3.1
     # via
     #   anyio

request-api/src/main.py

@@ -1,8 +1,9 @@
 import logging
 import os
-from datetime import datetime
+import datetime
 from typing import List, Dict, Any
 import sentry_sdk
+from slack_sdk import WebClient
 
 import boto3
 from botocore.exceptions import ClientError, BotoCoreError
@@ -43,17 +44,54 @@
 app = FastAPI()
 
 
+def send_slack_alert(message):
+    slack_token = os.environ.get("SLACK_BOT_TOKEN", "")
+    slack_channel = os.environ.get("SLACK_CHANNEL", "")
+    if not slack_token or not slack_channel:
+        logging.warning("Slack token or channel is missing.")
+        return
+    client = WebClient(token=slack_token)
+    bot_name = "SQS and DB"
+    client.chat_postMessage(channel=slack_channel, text=message, username=bot_name)
+
+
+def is_connection_restored(last_attempt_time, max_retry_duration=60):
+    current_time = datetime.now().timestamp()
+    last_attempt_timestamp = last_attempt_time.timestamp()
+    return (current_time - last_attempt_timestamp) > max_retry_duration
+
+
 # Dependency
 def _get_db():
-    db = session_maker()()
-    try:
-        yield db
-    finally:
-        db.close()
+    retries = 5
+    for attempt in range(retries):
+        try:
+            db = session_maker()()
+            try:
+                yield db
+            finally:
+                db.close()
+            return
+        except SQLAlchemyError as e:
+            logging.exception(f"Database connection failed (Attempt {attempt+1}): {e}")
+            if is_connection_restored(datetime.now()):
+                break
+    send_slack_alert("DB connection issue detected in async-request-backend..")
 
 
 def _get_sqs_client():
-    return boto3.client("sqs")
+    retries = 5
+    for attempt in range(retries):
+        try:
+            logging.info(f"SQS client successfully connected on attempt {attempt}.")
+            return boto3.client("sqs")
+        except (ClientError, BotoCoreError) as e:
+            logging.exception(
+                f"SQS connection failed on attempt {attempt}. Retrying..."
+            )
+            if is_connection_restored(datetime.now()):
+                break
+    send_slack_alert("SQS connection issue detected in async-request-backend..")
 
 
 @app.get("/health", response_model=HealthCheckResponse)
@@ -64,9 +102,7 @@ def healthcheck(
         db_result = db.execute(text("SELECT 1"))
         db_reachable = len(db_result.all()) == 1
     except SQLAlchemyError:
-        logging.exception(
-            "Health check of request-db failed",
-        )
+        logging.exception("Health check of request-db failed")
         db_reachable = False
 
     try: