Modularized backend and change in frontend

Author: ShivangNagta

.gitignore

@@ -0,0 +1,7 @@
+node_modules
+__pycache__
+models
+video_frames
+faiss_index.bin
+input_video.mp4
+metadata.pkl

CVPS.ipynb

@@ -206,7 +206,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 5,
+      "execution_count": null,
       "metadata": {
         "id": "bMdCOmne9_vT"
       },
@@ -294,6 +294,13 @@
         "frames = extract_frames(\"input_video.mp4\",\"video_frames\")"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": []
+    },
     {
       "cell_type": "code",
       "execution_count": null,

backend/.gitignore

@@ -0,0 +1,7 @@
+node_modules
+__pycache__
+models
+video_frames
+faiss_index.bin
+input_video.mp4
+metadata.pkl

backend/app.py

@@ -0,0 +1,4 @@
+import uvicorn
+
+if __name__ == "__main__":
+    uvicorn.run("app.main:app", host="0.0.0.0", port=8000, reload=True)

backend/app/__init__.py

@@ -0,0 +1,68 @@
+from fastapi import APIRouter, UploadFile, Request, File, Form, HTTPException
+from app.services.clip_model import clip_model
+from app.services.llava_api import query_llava, process_query
+from app.core.memory import ChatMemory
+from app.services.extract_frames import extract_frames
+from app.services.faiss import faiss_process
+
+router = APIRouter()
+memory = ChatMemory()
+
+from PIL import Image
+
+router = APIRouter()
+
+
+@router.post("/clip")
+async def clip_embed_video(file: UploadFile = File(...)):
+    """
+    Saves video, extracts frames, embeds them using CLIP, and stores in FAISS.
+    """
+    try:
+        # Step 1: Save uploaded video to disk
+        temp_path = "input_video.mp4"
+        with open(temp_path, "wb") as f:
+            f.write(await file.read())
+
+        # Step 2: Extract frames
+        frames = extract_frames(temp_path, "video_frames")
+        (preprocess, model) = clip_model()
+
+        faiss_process(preprocess, model, frames)
+        
+
+        return {"ready": True}
+    except Exception as e:
+        return {"ready": False, "error": str(e)}
+
+
+@router.post("/llava")
+async def llava_query(request: Request):
+    """
+    Accepts a question from JSON, retrieves relevant context from FAISS,
+    and sends it to Segmind's LLaVA API.
+    """
+    try:
+        body = await request.json()
+        question = body.get("question", "").strip()
+        print(f"Received question: {question}")
+
+        if not question:
+            raise HTTPException(status_code=400, detail="Question is required.")
+
+        # Step 1: Retrieve relevant context
+        (preprocess, model) = clip_model()
+        context_data = process_query(question, preprocess, model)
+        print(f"Retrieved context: {context_data}")
+
+        # Step 2: Query LLaVA
+        answer = query_llava(context_data, question)
+        # memory.add(question, answer)
+
+        print(f"LLaVA answer: {answer}")
+
+        return {"answer": answer}
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to get LLaVA response: {str(e)}")
+

backend/app/api/__init__.py

@@ -0,0 +1,14 @@
+from collections import deque
+
+class ChatMemory:
+    def __init__(self, max_len=10):
+        self.history = deque(maxlen=max_len)
+
+    def add(self, user_msg: str, bot_msg: str):
+        self.history.append({"user": user_msg, "bot": bot_msg})
+
+    def get(self):
+        return list(self.history)
+
+    def clear(self):
+        self.history.clear()

backend/app/api/routes.py

@@ -0,0 +1,16 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from app.api.routes import router  # Your routes
+
+app = FastAPI()
+
+# Allow requests from frontend (Vite dev server)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:5173"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(router)