add genai model to frontend

Author: ArmanpreetGhotra

client/package-lock.json

@@ -32,7 +32,8 @@
     "react-colorful": "^5.6.1",
     "react-dom": "^18.2.0",
     "sonner": "^2.0.5",
-    "tailwind-merge": "^3.2.0"
+    "tailwind-merge": "^3.2.0",
+    "@svgr/webpack": "^8.1.0"
   },
   "devDependencies": {
     "@eslint/eslintrc": "^3",

client/package.json

@@ -48,11 +48,20 @@ export default function WhiteBoard() {
     const textNodes = selectedNodes.filter(node => node.type === 'text');
     if (textNodes.length === 0) return;
 
-    const nodeToUpdate = textNodes[0]; // first selected text node
+    const nodeToUpdate = textNodes[0];
     const currentText = nodeToUpdate.data.label as string;
 
+    const endpointMap: Record<typeof action, string> = {
+      complete: 'completion',
+      summarize: 'summarization',
+      rephrase: 'rephrase'
+    };
+  
+    const endpoint = endpointMap[action];
+    const url = `http://localhost:8080/${endpoint}`;
+  
     try {
-      const response = await fetch(`http://localhost:8080/api/llm/rephrase`, {
+      const response = await fetch(url, {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({ user_text: [currentText] }),
@@ -67,11 +76,14 @@ export default function WhiteBoard() {
       setNodes((nds) =>
         nds.map((node) =>
           node.id === nodeToUpdate.id
-            ? { 
+            ? {
                 ...node,
-                data: { 
+                data: {
                   ...node.data,
-                  label: data.llm_response || '' // update node text from API response
+                  label:
+                    action === 'complete'
+                      ? `${currentText} ${data.llm_response || ''}`
+                      : data.llm_response || ''
                 }
               }
             : node
@@ -82,6 +94,7 @@ export default function WhiteBoard() {
     }
   };
 
+  
 
 
   return (

client/src/components/WhiteBoard.tsx

@@ -4,7 +4,7 @@ services:
     command:
       - "--providers.docker=true"
       - "--providers.docker.exposedByDefault=false"
-      - "--entrypoints.web.address=:80"
+      - "--entrypoints.web.address=:81"
       - "--entrypoints.websecure.address=:443"
       - "--entrypoints.web.http.redirections.entryPoint.to=websecure"
       - "--entrypoints.web.http.redirections.entryPoint.scheme=https"
@@ -14,7 +14,7 @@ services:
       - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json"
     restart: unless-stopped
     ports:
-      - "80:80"
+      - "81:81"
       - "443:443"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

compose.aws.yml

@@ -5,7 +5,7 @@ services:
     restart: always
     image: nginx:stable-alpine
     healthcheck:
-      test: "curl -f http://localhost:80 || exit 1"
+      test: "curl -f http://localhost:81 || exit 1"
       interval: 5s
     volumes:
       - ./docker/nginx/conf.d:/etc/nginx/conf.d:ro
@@ -14,7 +14,7 @@ services:
     depends_on:
       - server
     ports:
-      - "80:80"
+      - "81:81"
       - "443:443"
     networks:
       - server

docker-compose.yml

@@ -1,16 +1,158 @@
-from contextlib import asynccontextmanager
-from fastapi import FastAPI
-from app.routes import root
-from app.db.client import WeaviateClientSingleton
+import os
+import json
+import requests
+from typing import Dict, Any, List, Optional
+from fastapi import FastAPI, HTTPException, APIRouter
+from pydantic import BaseModel, Field
+from langchain.llms.base import LLM
+from langchain_core.prompts import PromptTemplate
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+import logging
+from fastapi.middleware.cors import CORSMiddleware
 
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-    WeaviateClientSingleton.get_instance()
-    print("Weaviate client initialized")
-    yield
+router = APIRouter()
 
+# Environment configuration
+CHAIR_API_KEY = "sk-a4370d697e294dac819c95ba5ffe0740"
+API_URL = "https://gpu.aet.cit.tum.de/api/chat/completions"
 
-app = FastAPI(lifespan=lifespan)
+class OpenWebUILLM(LLM):
+    api_url: str = API_URL
+    api_key: str = CHAIR_API_KEY
+    model_name: str = "llama3.3:latest"
+    
+    @property
+    def _llm_type(self) -> str:
+        return "open_webui"
+    
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> str:
+        if not self.api_key:
+            raise ValueError("CHAIR_API_KEY environment variable is required")
+        
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        
+        # Updated payload format
+        payload = {
+            "model": self.model_name,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": prompt
+                }
+            ]
+        }
+        
+        try:
+            logger.info(f"Sending request to OpenWebUI API: {payload}")
+            response = requests.post(
+                self.api_url,
+                headers=headers,
+                json=payload,
+                timeout=30
+            )
+            
+            if response.status_code != 200:
+                logger.error(f"API error: {response.status_code} - {response.text}")
+                raise requests.RequestException(f"API returned {response.status_code}: {response.text}")
+            
+            result = response.json()
+            logger.info(f"Received response: {result}")
+            
+            # Extract content from choices array
+            if "choices" in result and len(result["choices"]) > 0:
+                content = result["choices"][0]["message"]["content"]
+                return content.strip()
+            else:
+                logger.error(f"Unexpected response format: {result}")
+                raise ValueError(f"Unexpected response format: {result}")
+                
+        except requests.RequestException as e:
+            logger.error(f"API request failed: {str(e)}")
+            raise Exception(f"API request failed: {str(e)}")
+# Initialize FastAPI app
+app = FastAPI(
+    title="LLM Service",
+    description="OpenWebUI powered LLM service for text operations",
+    version="1.0.0"
+)
 
-app.include_router(root.router)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:3001", "http://localhost:8000/api/llm"],  
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Initialize LLM
+llm = OpenWebUILLM()
+
+class TextRequest(BaseModel):
+    user_text: List[str]
+
+class TextResponse(BaseModel):
+    llm_response: str
+
+@router.post("/completion", response_model=TextResponse)
+async def complete_text(request: TextRequest):
+    try:
+        prompt = f"""Complete the following text naturally:
+        {' '.join(request.user_text)}
+        
+        Provide a natural continuation that maintains the style and context in one or two sentences.
+        """
+        logger.info(f"Processing completion request for text: {request.user_text}")
+        result = llm(prompt)
+        logger.info(f"Generated completion: {result}")
+        return TextResponse(llm_response=result)
+    except Exception as e:
+        logger.error(f"Completion error: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.post("/summarization", response_model=TextResponse)
+async def summarize_text(request: TextRequest):
+    try:
+        prompt = f"""Summarize the following text concisely:
+        {' '.join(request.user_text)}
+        """
+        result = llm(prompt)
+        return TextResponse(llm_response=result)
+    except Exception as e:
+        logger.error(f"Summarization error: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.post("/rephrase", response_model=TextResponse)
+async def rephrase_text(request: TextRequest):
+    try:
+        logger.info(f"Received rephrase request: {request}")
+        prompt = f"""Rephrase the following text while maintaining its meaning in one or two sentences:
+        {' '.join(request.user_text)}
+        """
+        result = llm(prompt)
+        return TextResponse(llm_response=result)
+    except Exception as e:
+        logger.error(f"Rephrase error: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/health")
+async def health_check():
+    return {
+        "status": "healthy",
+        "model": llm.model_name,
+        "api_url": llm.api_url
+    }
+
+app.include_router(router)

genai/app/main.py

@@ -1,8 +1,107 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import List
+from fastapi.middleware.cors import CORSMiddleware
+import logging
+import requests
+from langchain.llms.base import LLM
+from langchain_core.prompts import PromptTemplate
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+from typing import Optional, Any
 
 router = APIRouter()
 
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
+# Constants
+CHAIR_API_KEY = "add api"
+API_URL = "https://gpu.aet.cit.tum.de/api/chat/completions"
+
+class OpenWebUILLM(LLM):
+    api_url: str = API_URL
+    api_key: str = CHAIR_API_KEY
+    model_name: str = "llama3.3:latest"
+
+    @property
+    def _llm_type(self) -> str:
+        return "open_webui"
+
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> str:
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        payload = {
+            "model": self.model_name,
+            "messages": [{"role": "user", "content": prompt}]
+        }
+
+        try:
+            logger.info(f"Sending request: {payload}")
+            response = requests.post(self.api_url, headers=headers, json=payload, timeout=30)
+            response.raise_for_status()
+            result = response.json()
+            content = result["choices"][0]["message"]["content"]
+            return content.strip()
+        except Exception as e:
+            logger.error(f"LLM call failed: {e}")
+            raise
+
+# LLM Instance
+llm = OpenWebUILLM()
+
+# Request/Response models
+class TextRequest(BaseModel):
+    user_text: List[str]
+
+class TextResponse(BaseModel):
+    llm_response: str
+
+# Routes
 @router.get("/")
 def read_root():
     return {"Hello": "World"}
+
+@router.post("/completion", response_model=TextResponse)
+async def complete_text(request: TextRequest):
+    prompt = f"Complete the following text naturally:\n{' '.join(request.user_text)}"
+    try:
+        result = llm(prompt)
+        return TextResponse(llm_response=result)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.post("/summarization", response_model=TextResponse)
+async def summarize_text(request: TextRequest):
+    prompt = f"Summarize the following text concisely:\n{' '.join(request.user_text)}"
+    try:
+        result = llm(prompt)
+        return TextResponse(llm_response=result)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.post("/rephrase", response_model=TextResponse)
+async def rephrase_text(request: TextRequest):
+    prompt = f"Rephrase the following text while maintaining its meaning:\n{' '.join(request.user_text)}"
+    try:
+        result = llm(prompt)
+        return TextResponse(llm_response=result)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/health")
+async def health_check():
+    return {
+        "status": "healthy",
+        "model": llm.model_name,
+        "api_url": llm.api_url
+    }