refactor: improve logging and code quality

surfsense_backend/app/routes/airtable_add_connector_route.py

@@ -24,6 +24,10 @@
 from app.schemas.airtable_auth_credentials import AirtableAuthCredentialsBase
 from app.users import current_active_user
 
+# Re-export for backward compatibility - this import is used by other modules
+# that import refresh_airtable_token from this module
+from app.utils.connector_auth import refresh_airtable_token as refresh_airtable_token  # noqa: F401
+
 logger = logging.getLogger(__name__)
 
 router = APIRouter()
@@ -286,78 +290,3 @@ async def airtable_callback(
         raise HTTPException(
             status_code=500, detail=f"Failed to complete Airtable OAuth: {e!s}"
         ) from e
-
-
-async def refresh_airtable_token(
-    session: AsyncSession, connector: SearchSourceConnector
-):
-    """
-    Refresh the Airtable access token for a connector.
-
-    Args:
-        session: Database session
-        connector: Airtable connector to refresh
-
-    Returns:
-        Updated connector object
-    """
-    try:
-        logger.info(f"Refreshing Airtable token for connector {connector.id}")
-
-        credentials = AirtableAuthCredentialsBase.from_dict(connector.config)
-        auth_header = make_basic_auth_header(
-            config.AIRTABLE_CLIENT_ID, config.AIRTABLE_CLIENT_SECRET
-        )
-
-        # Prepare token refresh data
-        refresh_data = {
-            "grant_type": "refresh_token",
-            "refresh_token": credentials.refresh_token,
-            "client_id": config.AIRTABLE_CLIENT_ID,
-            "client_secret": config.AIRTABLE_CLIENT_SECRET,
-        }
-
-        async with httpx.AsyncClient() as client:
-            token_response = await client.post(
-                TOKEN_URL,
-                data=refresh_data,
-                headers={
-                    "Content-Type": "application/x-www-form-urlencoded",
-                    "Authorization": auth_header,
-                },
-                timeout=30.0,
-            )
-
-        if token_response.status_code != 200:
-            raise HTTPException(
-                status_code=400, detail="Token refresh failed: {token_response.text}"
-            )
-
-        token_json = token_response.json()
-
-        # Calculate expiration time (UTC, tz-aware)
-        expires_at = None
-        if token_json.get("expires_in"):
-            now_utc = datetime.now(UTC)
-            expires_at = now_utc + timedelta(seconds=int(token_json["expires_in"]))
-
-        # Update credentials object
-        credentials.access_token = token_json["access_token"]
-        credentials.expires_in = token_json.get("expires_in")
-        credentials.expires_at = expires_at
-        credentials.scope = token_json.get("scope")
-
-        # Update connector config
-        connector.config = credentials.to_dict()
-        await session.commit()
-        await session.refresh(connector)
-
-        logger.info(
-            f"Successfully refreshed Airtable token for connector {connector.id}"
-        )
-
-        return connector
-    except Exception as e:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to refresh Airtable token: {e!s}"
-        ) from e

surfsense_backend/app/routes/chats_routes.py

@@ -1,7 +1,9 @@
+import logging
+
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import StreamingResponse
 from langchain.schema import AIMessage, HumanMessage
-from sqlalchemy.exc import IntegrityError, OperationalError
+from sqlalchemy.exc import IntegrityError, OperationalError, SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 from sqlalchemy.orm import selectinload
@@ -34,6 +36,8 @@
     validate_top_k,
 )
 
+logger = logging.getLogger(__name__)
+
 router = APIRouter()
 
 
@@ -180,19 +184,29 @@ async def create_chat(
         return db_chat
     except HTTPException:
         raise
-    except IntegrityError:
+    except IntegrityError as e:
         await session.rollback()
+        logger.warning("Chat creation failed due to integrity error: %s", e)
         raise HTTPException(
             status_code=400,
             detail="Database constraint violation. Please check your input data.",
         ) from None
-    except OperationalError:
+    except OperationalError as e:
         await session.rollback()
+        logger.error("Database operational error during chat creation: %s", e)
         raise HTTPException(
             status_code=503, detail="Database operation failed. Please try again later."
         ) from None
-    except Exception:
+    except SQLAlchemyError as e:
+        await session.rollback()
+        logger.error("Database error during chat creation: %s", e, exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail="An unexpected error occurred while creating the chat.",
+        ) from None
+    except Exception as e:
         await session.rollback()
+        logger.error("Unexpected error during chat creation: %s", e, exc_info=True)
         raise HTTPException(
             status_code=500,
             detail="An unexpected error occurred while creating the chat.",
@@ -266,11 +280,18 @@ async def read_chats(
         return result.all()
     except HTTPException:
         raise
-    except OperationalError:
+    except OperationalError as e:
+        logger.error("Database operational error while fetching chats: %s", e)
         raise HTTPException(
             status_code=503, detail="Database operation failed. Please try again later."
         ) from None
-    except Exception:
+    except SQLAlchemyError as e:
+        logger.error("Database error while fetching chats: %s", e, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail="An unexpected error occurred while fetching chats."
+        ) from None
+    except Exception as e:
+        logger.error("Unexpected error while fetching chats: %s", e, exc_info=True)
         raise HTTPException(
             status_code=500, detail="An unexpected error occurred while fetching chats."
         ) from None
@@ -308,11 +329,19 @@ async def read_chat(
         return chat
     except HTTPException:
         raise
-    except OperationalError:
+    except OperationalError as e:
+        logger.error("Database operational error while fetching chat %d: %s", chat_id, e)
         raise HTTPException(
             status_code=503, detail="Database operation failed. Please try again later."
         ) from None
-    except Exception:
+    except SQLAlchemyError as e:
+        logger.error("Database error while fetching chat %d: %s", chat_id, e, exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail="An unexpected error occurred while fetching the chat.",
+        ) from None
+    except Exception as e:
+        logger.error("Unexpected error while fetching chat %d: %s", chat_id, e, exc_info=True)
         raise HTTPException(
             status_code=500,
             detail="An unexpected error occurred while fetching the chat.",
@@ -357,19 +386,29 @@ async def update_chat(
         return db_chat
     except HTTPException:
         raise
-    except IntegrityError:
+    except IntegrityError as e:
         await session.rollback()
+        logger.warning("Chat update failed due to integrity error for chat %d: %s", chat_id, e)
         raise HTTPException(
             status_code=400,
             detail="Database constraint violation. Please check your input data.",
         ) from None
-    except OperationalError:
+    except OperationalError as e:
         await session.rollback()
+        logger.error("Database operational error while updating chat %d: %s", chat_id, e)
         raise HTTPException(
             status_code=503, detail="Database operation failed. Please try again later."
         ) from None
-    except Exception:
+    except SQLAlchemyError as e:
+        await session.rollback()
+        logger.error("Database error while updating chat %d: %s", chat_id, e, exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail="An unexpected error occurred while updating the chat.",
+        ) from None
+    except Exception as e:
         await session.rollback()
+        logger.error("Unexpected error while updating chat %d: %s", chat_id, e, exc_info=True)
         raise HTTPException(
             status_code=500,
             detail="An unexpected error occurred while updating the chat.",
@@ -407,18 +446,28 @@ async def delete_chat(
         return {"message": "Chat deleted successfully"}
     except HTTPException:
         raise
-    except IntegrityError:
+    except IntegrityError as e:
         await session.rollback()
+        logger.warning("Chat deletion failed due to integrity error for chat %d: %s", chat_id, e)
         raise HTTPException(
             status_code=400, detail="Cannot delete chat due to existing dependencies."
         ) from None
-    except OperationalError:
+    except OperationalError as e:
         await session.rollback()
+        logger.error("Database operational error while deleting chat %d: %s", chat_id, e)
         raise HTTPException(
             status_code=503, detail="Database operation failed. Please try again later."
         ) from None
-    except Exception:
+    except SQLAlchemyError as e:
+        await session.rollback()
+        logger.error("Database error while deleting chat %d: %s", chat_id, e, exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail="An unexpected error occurred while deleting the chat.",
+        ) from None
+    except Exception as e:
         await session.rollback()
+        logger.error("Unexpected error while deleting chat %d: %s", chat_id, e, exc_info=True)
         raise HTTPException(
             status_code=500,
             detail="An unexpected error occurred while deleting the chat.",

surfsense_backend/app/routes/podcasts_routes.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from pathlib import Path
 
@@ -26,6 +27,8 @@
 from app.users import current_active_user
 from app.utils.rbac import check_permission
 
+logger = logging.getLogger(__name__)
+
 router = APIRouter()
 
 
@@ -54,21 +57,24 @@ async def create_podcast(
         return db_podcast
     except HTTPException as he:
         raise he
-    except IntegrityError:
+    except IntegrityError as e:
         await session.rollback()
+        logger.warning("Podcast creation failed due to integrity error: %s", e)
         raise HTTPException(
             status_code=400,
             detail="Podcast creation failed due to constraint violation",
         ) from None
-    except SQLAlchemyError:
+    except SQLAlchemyError as e:
         await session.rollback()
+        logger.error("Database error while creating podcast: %s", e, exc_info=True)
         raise HTTPException(
             status_code=500, detail="Database error occurred while creating podcast"
         ) from None
-    except Exception:
+    except Exception as e:
         await session.rollback()
+        logger.error("Unexpected error while creating podcast: %s", e, exc_info=True)
         raise HTTPException(
-            status_code=500, detail="An unexpected error occurred"
+            status_code=500, detail="An unexpected error occurred while creating podcast"
         ) from None
 
 
@@ -115,10 +121,16 @@ async def read_podcasts(
         return result.scalars().all()
     except HTTPException:
         raise
-    except SQLAlchemyError:
+    except SQLAlchemyError as e:
+        logger.error("Database error while fetching podcasts: %s", e, exc_info=True)
         raise HTTPException(
             status_code=500, detail="Database error occurred while fetching podcasts"
         ) from None
+    except Exception as e:
+        logger.error("Unexpected error while fetching podcasts: %s", e, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail="An unexpected error occurred while fetching podcasts"
+        ) from None
 
 
 @router.get("/podcasts/{podcast_id}", response_model=PodcastRead)
@@ -153,10 +165,16 @@ async def read_podcast(
         return podcast
     except HTTPException as he:
         raise he
-    except SQLAlchemyError:
+    except SQLAlchemyError as e:
+        logger.error("Database error while fetching podcast: %s", e, exc_info=True)
         raise HTTPException(
             status_code=500, detail="Database error occurred while fetching podcast"
         ) from None
+    except Exception as e:
+        logger.error("Unexpected error while fetching podcast: %s", e, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail="An unexpected error occurred while fetching podcast"
+        ) from None
 
 
 @router.put("/podcasts/{podcast_id}", response_model=PodcastRead)
@@ -199,11 +217,18 @@ async def update_podcast(
         raise HTTPException(
             status_code=400, detail="Update failed due to constraint violation"
         ) from None
-    except SQLAlchemyError:
+    except SQLAlchemyError as e:
         await session.rollback()
+        logger.error("Database error while updating podcast: %s", e, exc_info=True)
         raise HTTPException(
             status_code=500, detail="Database error occurred while updating podcast"
         ) from None
+    except Exception as e:
+        await session.rollback()
+        logger.error("Unexpected error while updating podcast: %s", e, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail="An unexpected error occurred while updating podcast"
+        ) from None
 
 
 @router.delete("/podcasts/{podcast_id}", response_model=dict)
@@ -237,11 +262,18 @@ async def delete_podcast(
         return {"message": "Podcast deleted successfully"}
     except HTTPException as he:
         raise he
-    except SQLAlchemyError:
+    except SQLAlchemyError as e:
         await session.rollback()
+        logger.error("Database error while deleting podcast: %s", e, exc_info=True)
         raise HTTPException(
             status_code=500, detail="Database error occurred while deleting podcast"
         ) from None
+    except Exception as e:
+        await session.rollback()
+        logger.error("Unexpected error while deleting podcast: %s", e, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail="An unexpected error occurred while deleting podcast"
+        ) from None
 
 
 async def generate_chat_podcast_with_new_session(
@@ -260,9 +292,7 @@ async def generate_chat_podcast_with_new_session(
                 session, chat_id, search_space_id, user_id, podcast_title, user_prompt
             )
         except Exception as e:
-            import logging
-
-            logging.error(f"Error generating podcast from chat: {e!s}")
+            logger.error("Error generating podcast from chat: %s", e, exc_info=True)
 
 
 @router.post("/podcasts/generate")

surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py

@@ -8,8 +8,8 @@
 from app.config import config
 from app.connectors.airtable_connector import AirtableConnector
 from app.db import Document, DocumentType, SearchSourceConnectorType
-from app.routes.airtable_add_connector_route import refresh_airtable_token
 from app.schemas.airtable_auth_credentials import AirtableAuthCredentialsBase
+from app.utils.connector_auth import refresh_airtable_token
 from app.services.llm_service import get_user_long_context_llm
 from app.services.task_logging_service import TaskLoggingService
 from app.utils.document_converters import (