PostgreSQL database integration for conversation persistence (#181)

* chore: update deps
* feat: add session id to input object
* feat: implement database models and CRUD operations
* feat: add conversations API with DB integration
* feat: add api docs
* chore: move api docs
* feat: delete unused chains api
* feat: add database configuration to .env.example
* chore: update deps
* feat: update dockerfile
* feat: Add postgresql and pgadmin services
* feat: add response models for conversations endpoints
* chore: update deps
* feat: include conversations router
* feat: initialize database module
* fix: Expose core components in package init
* feat: load environment variables and configure CUDA settings
* chore: update deps
* fix: update embeddings type
* refactor: move CUDA environment variable setup to main entry point
* fix: update healthcheck command to include database name
* refactor: replace session_id with conversation_id in db models
* refactor: update type hints
* refactor: formatting changes
* fix: handle incomplete message pairs
* fix: remove hardcoded PostgreSQL environment variables
* feat: update psycopg2 dependency to psycopg2-binary
* fix: update import from graphs to conversations
* chore: update google-cloud-storage dependency
* fix: use retriever graph
* fix: revert google-cloud-storage dependency
* fix: add return type hints
* fix: add return type hint to lifespan function
* docs: add docstrings for endpoints
* fix: untrack data folder
* docs: update README with PostgreSQL setup instructions
* remove api_docs
* remove unused response_models
* db models:
- use uuid
- standardise var names
* specific docker-compose commands
* improve history string robustness
* fix LLM response extraction (docker compose)
- async bug `get_tools`
- pass chat history in prompt
* add unit tests
* reduce docstring verbosity
* feat: save streamed conversation messages to the database
* fix streaming for more messages
* add streaming unit tests
* fix lint
* fix test

---------

Signed-off-by: Palaniappan R <[email protected]>
Signed-off-by: Jack Luar <[email protected]>
Co-authored-by: Jack Luar <[email protected]>

Author: palaniappan-r

.gitignore

@@ -1,7 +1,7 @@
 .env
 *.ipynb
 __pycache__/
-backend/data/*
+backend/data/
 backend/src/*.json
 *.pyc
 *.egg-info/

Makefile

@@ -23,11 +23,15 @@ check:
 
 .PHONY: docker-up
 docker-up:
-	@docker compose up --build --wait
+	@docker compose -f docker-compose.yml up --build --wait
 
 .PHONY: docker-down
 docker-down:
-	@docker compose down --remove-orphans
+	@docker compose -f docker-compose.yml down --remove-orphans
+
+.PHONY: docker-dev
+docker-dev:
+	@docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build --wait
 
 # --- Development Commands ---
 .PHONY: seed-credentials

README.md

@@ -27,6 +27,31 @@ This setup involves the setting of both the frontend and backend components. We
 
 ### Backend Setup
 
+#### Database Schema
+
+The database automatically creates the following tables:
+- `conversations` - Stores conversation metadata (id, user_id, title, timestamps)
+- `messages` - Stores individual messages within conversations
+
+#### Setting Up PostgreSQL Database Variables
+
+The backend uses PostgreSQL for conversation persistence. Configure these database variables in your `.env` file:
+
+- `POSTGRES_USER` - Database username (default: `orassistant`)
+- `POSTGRES_PASSWORD` - Database password (default: `password`)
+- `POSTGRES_DB` - Database name (default: `orassistant_db`)
+- `POSTGRES_HOST` - Database host (default: `postgres` for Docker, `localhost` for local)
+- `POSTGRES_PORT` - Database port (default: `5432`)
+
+**For local development without Docker:**
+1. Install PostgreSQL on your system
+2. Create a database: `createdb orassistant_db`
+3. Set `POSTGRES_HOST=localhost` in your `.env` file
+
+**For Docker deployment:**
+- The database is automatically configured via docker-compose
+- Data persists in a Docker volume named `postgres_data`
+
 #### Option 1 - Docker
 
 Ensure you have `docker` and `docker-compose` installed in your system.

backend/.env.example

@@ -22,7 +22,7 @@ GOOGLE_GEMINI=2.0_flash
 
 LLM_TEMP=1
 
-EMBEDDINGS_TYPE=GOOGLE_VERTEXAI
+EMBEDDINGS_TYPE=GOOGLE_GENAI
 GOOGLE_EMBEDDINGS=text-embedding-004
 HF_EMBEDDINGS=thenlper/gte-large
 HF_RERANKER=BAAI/bge-reranker-base
@@ -70,3 +70,10 @@ HEALTHCHECK_INTERVAL=30s
 HEALTHCHECK_TIMEOUT=10s
 HEALTHCHECK_RETRIES=5
 HEALTHCHECK_START_PERIOD=1200s
+
+# PostgreSQL Database Configuration
+POSTGRES_USER=orassistant
+POSTGRES_PASSWORD=password
+POSTGRES_DB=orassistant_db
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432

backend/Dockerfile

@@ -2,7 +2,18 @@ FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
 
 WORKDIR /ORAssistant-backend
 
-RUN apt-get update && apt-get install -y pandoc git wget curl git-lfs && git lfs install
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    gcc \
+    git \
+    git-lfs \
+    libpq-dev \
+    pandoc \
+    postgresql-client \
+    wget && \
+    git lfs install && \
+    rm -rf /var/lib/apt/lists/*
 
 RUN pip install uv
 

backend/Dockerfile_slim

@@ -3,10 +3,18 @@ FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
 WORKDIR /ORAssistant-backend
 
 RUN apt-get update && apt-get install -y \
-    pandoc git wget curl \
-    git-lfs
+    build-essential \
+    curl \
+    gcc \
+    git \
+    git-lfs \
+    libpq-dev \
+    pandoc \
+    postgresql-client \
+    wget && \
+    git lfs install && \
+    rm -rf /var/lib/apt/lists/*
 
-RUN git lfs install
 RUN pip install uv
 
 COPY ./pyproject.toml /ORAssistant-backend/pyproject.toml

backend/chatbot.py

@@ -1,6 +1,6 @@
 import os
 import logging
-from src.api.routers import graphs
+from src.api.routers import conversations
 
 
 def get_history_str(chat_history: list[dict[str, str]]) -> str:
@@ -13,7 +13,7 @@ def get_history_str(chat_history: list[dict[str, str]]) -> str:
 chat_history: list[dict[str, str]] = []
 
 if __name__ == "__main__":
-    rg = graphs.rg
+    rg = conversations.rg
     os.system("clear")
 
     while True:

backend/main.py

@@ -2,10 +2,13 @@
 import uvicorn
 from dotenv import load_dotenv
 
-from src.api.main import app
-
 load_dotenv()
 
+if os.getenv("USE_CUDA", "false").lower() != "true":
+    os.environ["CUDA_VISIBLE_DEVICES"] = ""
+
+from src.api.main import app  # noqa: E402
+
 
 def main() -> None:
     uvicorn.run(

backend/pyproject.toml

@@ -5,9 +5,12 @@ description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.13"
 dependencies = [
+    "asyncpg>=0.30.0",
     "faiss-cpu==1.12.0",
     "fastapi==0.116.1",
     "fastmcp>=2.12.2",
+    "google-cloud-storage>=2.19.0",
+    "httpx>=0.28.1",
     "huggingface-hub[cli]==0.34.4",
     "langchain==0.3.27",
     "langchain-community==0.3.27",
@@ -19,17 +22,22 @@ dependencies = [
     "langgraph==0.6.6",
     "markdown==3.8.2",
     "myst-parser==4.0.1",
+    "nest-asyncio>=1.6.0",
     "nltk==3.9.1",
     "openai==1.100.2",
-    "pypdf==6.1.3",
+    "psycopg2-binary>=2.9.11",
+    "pydantic>=2.11.7",
+    "pypdf==6.0.0",
     "rank-bm25==0.2.2",
+    "rich>=13.7.0",
     "sentence-transformers>=5.1.0",
     "sphinx==8.1.3",
     "sphinx-autobuild==2024.10.3",
     "sphinx-book-theme==1.1.4",
     "sphinx-copybutton==0.5.2",
     "sphinx-external-toc==1.0.1",
     "sphinxcontrib-mermaid==1.0.0",
+    "sqlalchemy>=2.0.43",
     "unstructured==0.18.13",
 ]
 
@@ -58,6 +66,10 @@ markers = [
 ]
 asyncio_mode = "auto"
 
+[[tool.mypy.overrides]]
+module = "nest_asyncio"
+ignore_missing_imports = true
+
 [tool.mypy]
 exclude = [
     "^tests/",

backend/src/agents/retriever_graph.py

@@ -106,12 +106,21 @@ def classify(self, state: AgentState) -> dict[str, list[str]]:
             return {"agent_type": [response.content]}  # type: ignore
 
     def fork_route(self, state: AgentState) -> str:
-        # TODO: if more than one agent add handler
-        if not self.enable_mcp:
-            tmp = "rag_agent"
-        else:
-            tmp = "mcp_agent"
-        return tmp
+        """Route to the appropriate agent based on classification"""
+        agent_type = state.get("agent_type", [])
+
+        if not agent_type:
+            # Default to RAG agent if no classification
+            return "rag_agent"
+
+        # Return the classified agent type
+        classified = agent_type[0]
+
+        # If MCP is disabled but classifier chose MCP, fall back to RAG
+        if classified == "mcp_agent" and not self.enable_mcp:
+            return "rag_agent"
+
+        return classified
 
     def initialize(self) -> None:
         self.workflow = StateGraph(AgentState)