Feature: backend add pgvector and document chunking to DB

.github/workflows/deploy-production.yml

@@ -24,6 +24,11 @@ jobs:
       SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
       EMAILS_FROM_EMAIL: ${{ secrets.EMAILS_FROM_EMAIL }}
       POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
     steps:
       - name: Checkout

.github/workflows/deploy-staging.yml

@@ -24,6 +24,11 @@ jobs:
       SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
       EMAILS_FROM_EMAIL: ${{ secrets.EMAILS_FROM_EMAIL }}
       POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
     steps:
       - name: Checkout

.github/workflows/generate-client.yml

@@ -11,6 +11,12 @@ jobs:
     permissions:
       contents: write
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
     steps:
     # For PRs from forks
     - uses: actions/checkout@v4

.github/workflows/playwright.yml

@@ -18,6 +18,12 @@ on:
 jobs:
   changes:
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
     # Set job outputs to values from filter step
     outputs:
       changed: ${{ steps.filter.outputs.changed }}
@@ -41,6 +47,11 @@ jobs:
     if: ${{ needs.changes.outputs.changed == 'true' }}
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     strategy:
       matrix:
         shardIndex: [1, 2, 3, 4]
@@ -92,6 +103,11 @@ jobs:
     # Merge reports after playwright-tests, even if some shards have failed
     if: ${{ !cancelled() && needs.changes.outputs.changed == 'true' }}
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-node@v4
@@ -123,6 +139,11 @@ jobs:
     needs:
       - test-playwright
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         uses: re-actors/alls-green@release/v1

.github/workflows/test-backend.yml

@@ -12,6 +12,12 @@ on:
 jobs:
   test-backend:
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/test-docker-compose.yml

@@ -13,14 +13,36 @@ jobs:
 
   test-docker-compose:
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - run: docker compose build
-      - run: docker compose down -v --remove-orphans
-      - run: docker compose up -d --wait backend frontend adminer
+      - name: Create .env for Docker Compose
+        run: |
+          echo "POSTGRES_USER=$POSTGRES_USER" >> backend/.env
+          echo "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" >> backend/.env
+          echo "POSTGRES_DB=$POSTGRES_DB" >> backend/.env
+          echo "POSTGRES_PORT=$POSTGRES_PORT" >> backend/.env
+          echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> backend/.env
+          echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> backend/.env
+          echo "AWS_REGION=$AWS_REGION" >> backend/.env
+          echo "S3_BUCKET=$S3_BUCKET" >> backend/.env
+          echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> backend/.env
+          echo "STACK_NAME=study-assistant" >> backend/.env
+
+
+      - run: docker compose --env-file backend/.env build
+      - run: docker compose --env-file backend/.env down -v --remove-orphans
+      - run: docker compose --env-file backend/.env up -d --wait backend adminer
+
       - name: Test backend is up
         run: curl http://localhost:8000/api/v1/utils/health-check
-      - name: Test frontend is up
-        run: curl http://localhost:5173
-      - run: docker compose down -v --remove-orphans
+      # - name: Test frontend is up
+      #   run: curl http://localhost:5173
+      # - run: docker compose down -v --remove-orphans

.gitignore

@@ -4,3 +4,4 @@ node_modules/
 /playwright-report/
 /blob-report/
 /playwright/.cache/
+.env

.pre-commit-config.yaml

@@ -17,14 +17,32 @@ repos:
             )$
       - id: trailing-whitespace
         exclude: ^frontend/src/client/.*
+  # - repo: https://github.com/pre-commit/mirrors-eslint
+  #   rev: v8.53.0
+  #   hooks:
+  #     - id: eslint
+  #       files: \.[jt]sx?$
+  #       types: [file]
+  #       additional_dependencies:
+  #         - [email protected]
+  #         - [email protected]
+  #         - [email protected]
+  #         - "@typescript-eslint/[email protected]"
+  #         - "@typescript-eslint/[email protected]"
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     rev: v0.2.2
     hooks:
       - id: ruff
         args:
           - --fix
       - id: ruff-format
-
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.1.0
+    hooks:
+      - id: prettier
+        files: \.(ts|tsx)$
+        additional_dependencies:
+          - [email protected]
 ci:
   autofix_commit_msg: 🎨 [pre-commit.ci] Auto format from pre-commit.com hooks
   autoupdate_commit_msg: ⬆ [pre-commit.ci] pre-commit autoupdate

backend/app/api/main.py

@@ -1,14 +1,14 @@
 from fastapi import APIRouter
 
-from app.api.routes import items, login, private, users, utils
+from app.api.routes import documents, exams, login, private, users, utils
 from app.core.config import settings
 
 api_router = APIRouter()
 api_router.include_router(login.router)
 api_router.include_router(users.router)
 api_router.include_router(utils.router)
-api_router.include_router(items.router)
-
+api_router.include_router(documents.router)
+api_router.include_router(exams.router)
 
 if settings.ENVIRONMENT == "local":
     api_router.include_router(private.router)

backend/app/api/routes/documents.py

@@ -0,0 +1,145 @@
+import uuid
+from typing import Any
+
+from fastapi import APIRouter, BackgroundTasks, File, HTTPException, UploadFile
+from sqlmodel import func, select
+
+from app.api.deps import CurrentUser, SessionDep
+from app.core.extractors import extract_text_and_save_to_db
+from app.models import (
+    Document,
+    DocumentCreate,
+    DocumentPublic,
+    DocumentsPublic,
+    DocumentUpdate,
+    Message,
+)
+from app.s3 import generate_s3_url, upload_file_to_s3
+
+router = APIRouter(prefix="/documents", tags=["documents"])
+
+
+@router.post("/", response_model=DocumentPublic)
+def create_document(
+    *,
+    session: SessionDep,
+    current_user: CurrentUser,
+    background_tasks: BackgroundTasks,  # noqa: ARG001
+    file: UploadFile = File(...),
+) -> Any:
+    key = None
+    try:
+        key = upload_file_to_s3(file, str(current_user.id))
+    except Exception as e:
+        raise HTTPException(500, f"Failed to upload file. Error: {str(e)}")
+
+    try:
+        url = generate_s3_url(key)
+    except Exception:
+        raise HTTPException(500, f"Could not generate URL for file key: {key}")
+
+    document_in = DocumentCreate(
+        filename=file.filename,
+        content_type=file.content_type,
+        size=file.size,
+        s3_url=url,
+        s3_key=key,
+    )
+
+    document = Document.model_validate(
+        document_in, update={"owner_id": current_user.id}
+    )
+
+    session.add(document)
+    session.commit()
+    session.refresh(document)
+
+    # 3. Kick off background job
+    print("Document created, starting background task...")
+    background_tasks.add_task(extract_text_and_save_to_db, key, str(document.id))
+    return document
+
+
+@router.get("/{id}", response_model=DocumentPublic)
+def read_document(session: SessionDep, current_user: CurrentUser, id: uuid.UUID) -> Any:
+    """
+    Get document by ID.
+    """
+    document = session.get(Document, id)
+    if not document:
+        raise HTTPException(status_code=404, detail="Document not found")
+    if not current_user.is_superuser and (document.owner_id != current_user.id):
+        raise HTTPException(status_code=400, detail="Not enough permissions")
+    return document
+
+
+@router.get("/", response_model=DocumentsPublic)
+def read_documents(
+    session: SessionDep, current_user: CurrentUser, skip: int = 0, limit: int = 100
+) -> Any:
+    """
+    Retrieve documents.
+    """
+
+    if current_user.is_superuser:
+        count_statement = select(func.count()).select_from(Document)
+        count = session.exec(count_statement).one()
+        statement = select(Document).offset(skip).limit(limit)
+        documents = session.exec(statement).all()
+    else:
+        count_statement = (
+            select(func.count())
+            .select_from(Document)
+            .where(Document.owner_id == current_user.id)
+        )
+        count = session.exec(count_statement).one()
+        statement = (
+            select(Document)
+            .where(Document.owner_id == current_user.id)
+            .offset(skip)
+            .limit(limit)
+        )
+        documents = session.exec(statement).all()
+
+    return DocumentsPublic(data=documents, count=count)
+
+
+@router.put("/{id}", response_model=DocumentPublic)
+def update_document(
+    *,
+    session: SessionDep,
+    current_user: CurrentUser,
+    id: uuid.UUID,
+    document_in: DocumentUpdate,
+) -> Any:
+    """
+    Update an document.
+    """
+    document = session.get(Document, id)
+    if not document:
+        raise HTTPException(status_code=404, detail="Document not found")
+    if not current_user.is_superuser and (document.owner_id != current_user.id):
+        raise HTTPException(status_code=400, detail="Not enough permissions")
+    update_dict = document_in.model_dump(exclude_unset=True)
+    document.sqlmodel_update(update_dict)
+    session.add(document)
+    session.commit()
+    session.refresh(document)
+    return document
+
+
+@router.delete("/{id}")
+def delete_document(
+    session: SessionDep, current_user: CurrentUser, id: uuid.UUID
+) -> Message:
+    """
+    Delete an document.
+    """
+    document = session.get(Document, id)
+    if not document:
+        raise HTTPException(status_code=404, detail="Document not found")
+    if not current_user.is_superuser and (document.owner_id != current_user.id):
+        raise HTTPException(status_code=400, detail="Not enough permissions")
+    session.delete(document)
+    session.commit()
+    return Message(message="Document deleted successfully")