Bug: remove frontend item refs

.github/workflows/deploy-production.yml

@@ -24,6 +24,10 @@ jobs:
       SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
       EMAILS_FROM_EMAIL: ${{ secrets.EMAILS_FROM_EMAIL }}
       POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
     steps:
       - name: Checkout

.github/workflows/deploy-staging.yml

@@ -24,6 +24,10 @@ jobs:
       SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
       EMAILS_FROM_EMAIL: ${{ secrets.EMAILS_FROM_EMAIL }}
       POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
     steps:
       - name: Checkout

.github/workflows/generate-client.yml

@@ -11,6 +11,11 @@ jobs:
     permissions:
       contents: write
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     steps:
     # For PRs from forks
     - uses: actions/checkout@v4

.github/workflows/playwright.yml

@@ -18,6 +18,11 @@ on:
 jobs:
   changes:
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     # Set job outputs to values from filter step
     outputs:
       changed: ${{ steps.filter.outputs.changed }}
@@ -41,6 +46,11 @@ jobs:
     if: ${{ needs.changes.outputs.changed == 'true' }}
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     strategy:
       matrix:
         shardIndex: [1, 2, 3, 4]
@@ -92,6 +102,11 @@ jobs:
     # Merge reports after playwright-tests, even if some shards have failed
     if: ${{ !cancelled() && needs.changes.outputs.changed == 'true' }}
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-node@v4
@@ -123,6 +138,11 @@ jobs:
     needs:
       - test-playwright
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         uses: re-actors/alls-green@release/v1

.github/workflows/test-backend.yml

@@ -12,6 +12,11 @@ on:
 jobs:
   test-backend:
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/test-docker-compose.yml

@@ -13,6 +13,11 @@ jobs:
 
   test-docker-compose:
     runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.gitignore

@@ -4,3 +4,4 @@ node_modules/
 /playwright-report/
 /blob-report/
 /playwright/.cache/
+.env

backend/app/api/main.py

@@ -1,13 +1,13 @@
 from fastapi import APIRouter
 
-from app.api.routes import items, login, private, users, utils
+from app.api.routes import documents, login, private, users, utils
 from app.core.config import settings
 
 api_router = APIRouter()
 api_router.include_router(login.router)
 api_router.include_router(users.router)
 api_router.include_router(utils.router)
-api_router.include_router(items.router)
+api_router.include_router(documents.router)
 
 
 if settings.ENVIRONMENT == "local":

backend/app/api/routes/documents.py

@@ -0,0 +1,50 @@
+from typing import Any
+
+from fastapi import APIRouter, BackgroundTasks, File, HTTPException, UploadFile
+
+from app.api.deps import CurrentUser, SessionDep
+from app.core.extractors import extract_text_and_save_to_db
+from app.models import Document, DocumentCreate, DocumentPublic
+from app.s3 import generate_s3_url, upload_file_to_s3
+
+router = APIRouter(prefix="/documents", tags=["documents"])
+
+
+@router.post("/", response_model=DocumentPublic)
+def create_document(
+    *,
+    session: SessionDep,
+    current_user: CurrentUser,
+    background_tasks: BackgroundTasks,  # noqa: ARG001
+    file: UploadFile = File(...),
+) -> Any:
+    key = None
+    try:
+        key = upload_file_to_s3(file, str(current_user.id))
+    except Exception as e:
+        raise HTTPException(500, f"Failed to upload file. Error: {str(e)}")
+
+    try:
+        url = generate_s3_url(key)
+    except Exception:
+        raise HTTPException(500, f"Could not generate URL for file key: {key}")
+
+    document_in = DocumentCreate(
+        filename=file.filename,
+        content_type=file.content_type,
+        size=file.size,
+        s3_url=url,
+    )
+
+    document = Document.model_validate(
+        document_in, update={"owner_id": current_user.id}
+    )
+
+    session.add(document)
+    session.commit()
+    session.refresh(document)
+
+    # 3. Kick off background job
+    print("Document created, starting background task...")
+    background_tasks.add_task(extract_text_and_save_to_db, url, str(document.id))
+    return document

backend/app/api/routes/users.py

@@ -13,7 +13,7 @@
 from app.core.config import settings
 from app.core.security import get_password_hash, verify_password
 from app.models import (
-    Item,
+    Document,
     Message,
     UpdatePassword,
     User,
@@ -219,7 +219,7 @@ def delete_user(
         raise HTTPException(
             status_code=403, detail="Super users are not allowed to delete themselves"
         )
-    statement = delete(Item).where(col(Item.owner_id) == user_id)
+    statement = delete(Document).where(col(Document.owner_id) == user_id)
     session.exec(statement)  # type: ignore
     session.delete(user)
     session.commit()

backend/app/core/config.py

@@ -57,6 +57,11 @@ def all_cors_origins(self) -> list[str]:
     POSTGRES_PASSWORD: str = ""
     POSTGRES_DB: str = ""
 
+    AWS_ACCESS_KEY_ID: str = ""
+    AWS_SECRET_ACCESS_KEY: str = ""
+    AWS_REGION: str = ""
+    S3_BUCKET_NAME: str = ""
+
     @computed_field  # type: ignore[prop-decorator]
     @property
     def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn:

backend/app/core/db.py

@@ -1,4 +1,4 @@
-from sqlmodel import Session, create_engine, select
+from sqlmodel import Session, SQLModel, create_engine, select
 
 from app import crud
 from app.core.config import settings
@@ -16,10 +16,9 @@ def init_db(session: Session) -> None:
     # Tables should be created with Alembic migrations
     # But if you don't want to use migrations, create
     # the tables un-commenting the next lines
-    # from sqlmodel import SQLModel
 
     # This works because the models are already imported and registered from app.models
-    # SQLModel.metadata.create_all(engine)
+    SQLModel.metadata.create_all(engine)
 
     user = session.exec(
         select(User).where(User.email == settings.FIRST_SUPERUSER)

backend/app/core/extractors.py

@@ -0,0 +1,27 @@
+import os
+import tempfile
+from app.core.db import engine
+from sqlmodel import Session, select
+import textract
+import requests
+from app.models import Document
+from app.s3 import extract_text_from_s3_file
+
+def extract_text_and_save_to_db(s3_url: str, document_id: str) -> None:
+    try:
+        with Session(engine) as session:
+            text = extract_text_from_s3_file(s3_url)
+
+            document_query = select(Document).where(Document.id == document_id)
+            document = session.exec(document_query).first()
+
+            if not document:
+                raise Exception(f"Document with ID {document_id} not found")
+
+            document.extracted_text = text
+            session.add(document)
+            session.commit()
+
+    except Exception as e:
+        print(f"Failed to extract and chunk text for document {document_id}: {e}")
+