added image objects to api call to agent for better context for agent

Author: codinglabsong

api/llm/agent.py

@@ -65,15 +65,15 @@ def get_agent():
 
 
 def chat_with_agent(
-    message: str, user_id: str = "default", selected_images: Optional[List[str]] = None
+    message: str, user_id: str = "default", selected_images: Optional[List[dict]] = None
 ) -> str:
     """
     Send a message to the agent and get a response.
 
     Args:
         message: The user's message
         user_id: Unique identifier for the user/thread
-        selected_images: List of selected image names (optional)
+        selected_images: List of selected image objects (optional)
 
     Returns:
         The agent's response as a string
@@ -83,7 +83,16 @@ def chat_with_agent(
     # Prepare the message with context
     full_message = message
     if selected_images and len(selected_images) > 0:
-        image_context = f" Selected images: {', '.join(selected_images)}."
+        image_context = "\n\nSelected Images:\n"
+        for i, img in enumerate(selected_images, 1):
+            image_context += (
+                f"{i}. {img.get('title', 'Untitled')} (ID: {img.get('id', 'unknown')})\n"
+            )
+            image_context += f"   Type: {img.get('type', 'unknown')}\n"
+            image_context += f"   Description: {img.get('description', 'No description')}\n"
+            if img.get("url"):
+                image_context += f"   URL: {img.get('url')}\n"
+            image_context += "\n"
         full_message = message + image_context
 
     # Configure thread ID for conversation continuity

api/server/main.py

@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
@@ -14,7 +14,7 @@
 
 class ChatRequest(BaseModel):
     message: str
-    selected_images: Optional[List[str]] = []
+    selected_images: Optional[List[Dict[str, Any]]] = []
     user_id: Optional[str] = None
 
 

src/app/page.tsx

@@ -101,13 +101,12 @@ export default function Home() {
 
   const handleChatMessage = async (message: string) => {
     const selectedImageIds = Array.from(selectedImages);
-    const imageMetadata =
-      selectedImageIds.length > 0
-        ? `\n\nSelected Images: ${selectedImageIds.join(", ")}`
-        : "";
+    const selectedImageObjects = images.filter((img) =>
+      selectedImageIds.includes(img.id),
+    );
 
-    // Add user message
-    const userMessage = createMessage(message + imageMetadata, "user");
+    // Add user message (without image metadata)
+    const userMessage = createMessage(message, "user");
     addMessage(userMessage);
 
     // Set loading state
@@ -117,7 +116,7 @@ export default function Home() {
     try {
       const apiResponse = await sendChatMessage({
         message,
-        selected_images: selectedImageIds,
+        selected_images: selectedImageObjects,
         user_id: userId,
       });
 

src/lib/actions.ts

@@ -9,7 +9,14 @@ import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 
 interface ChatRequest {
   message: string;
-  selected_images: string[];
+  selected_images: Array<{
+    id: string;
+    url: string | null;
+    title: string;
+    description: string;
+    timestamp: Date;
+    type: "uploaded" | "generated" | "sample";
+  }>;
   user_id?: string;
 }