Merge pull request #77 from jakobhoeg/development

feat: add vision support (dev env. support only for now)

Author: jakobhoeg

package-lock.json

@@ -9,56 +9,59 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@emoji-mart/data": "^1.1.2",
+    "@emoji-mart/data": "^1.2.1",
     "@emoji-mart/react": "^1.1.1",
-    "@hookform/resolvers": "^3.3.4",
-    "@langchain/community": "^0.0.26",
-    "@langchain/core": "^0.1.35",
-    "@radix-ui/react-avatar": "^1.0.4",
-    "@radix-ui/react-dialog": "^1.0.5",
-    "@radix-ui/react-dropdown-menu": "^2.0.6",
+    "@hookform/resolvers": "^3.9.0",
+    "@langchain/community": "^0.3.1",
+    "@langchain/core": "^0.3.3",
+    "@radix-ui/react-avatar": "^1.1.0",
+    "@radix-ui/react-dialog": "^1.1.1",
+    "@radix-ui/react-dropdown-menu": "^2.1.1",
     "@radix-ui/react-icons": "^1.3.0",
-    "@radix-ui/react-label": "^2.0.2",
-    "@radix-ui/react-popover": "^1.0.7",
-    "@radix-ui/react-scroll-area": "^1.0.5",
-    "@radix-ui/react-select": "^2.0.0",
-    "@radix-ui/react-slot": "^1.0.2",
-    "@radix-ui/react-tooltip": "^1.0.7",
+    "@radix-ui/react-label": "^2.1.0",
+    "@radix-ui/react-popover": "^1.1.1",
+    "@radix-ui/react-scroll-area": "^1.1.0",
+    "@radix-ui/react-select": "^2.1.1",
+    "@radix-ui/react-slot": "^1.1.0",
+    "@radix-ui/react-tooltip": "^1.1.2",
     "@types/dom-speech-recognition": "^0.0.4",
-    "ai": "^2.2.37",
+    "ai": "^3.4.0",
     "class-variance-authority": "^0.7.0",
-    "clsx": "^2.1.0",
-    "emoji-mart": "^5.5.2",
-    "framer-motion": "^11.0.3",
-    "langchain": "^0.1.13",
-    "lucide-react": "^0.322.0",
-    "next": "^14.2.3",
-    "next-themes": "^0.2.1",
-    "react": "^18",
+    "clsx": "^2.1.1",
+    "emoji-mart": "^5.6.0",
+    "framer-motion": "^11.5.6",
+    "langchain": "^0.3.2",
+    "lucide-react": "^0.445.0",
+    "next": "^14.2.13",
+    "next-themes": "^0.3.0",
+    "ollama-ai-provider": "^0.15.0",
+    "react": "^18.3.1",
     "react-code-blocks": "^0.1.6",
-    "react-dom": "^18",
-    "react-hook-form": "^7.50.1",
+    "react-dom": "^18.3.1",
+    "react-dropzone": "^14.2.9",
+    "react-hook-form": "^7.53.0",
     "react-markdown": "^9.0.1",
-    "react-resizable-panels": "^2.0.3",
+    "react-resizable-panels": "^2.1.3",
     "react-textarea-autosize": "^8.5.3",
     "remark-gfm": "^4.0.0",
-    "sharp": "^0.33.4",
-    "sonner": "^1.4.0",
-    "tailwind-merge": "^2.2.1",
+    "sharp": "^0.33.5",
+    "sonner": "^1.5.0",
+    "tailwind-merge": "^2.5.2",
     "tailwindcss-animate": "^1.0.7",
-    "uuid": "^9.0.1",
-    "zod": "^3.22.4"
+    "uuid": "^10.0.0",
+    "zod": "^3.23.8",
+    "zustand": "^5.0.0-rc.2"
   },
   "devDependencies": {
-    "@types/node": "^20",
-    "@types/react": "^18",
-    "@types/react-dom": "^18",
-    "@types/uuid": "^9.0.8",
-    "autoprefixer": "^10.0.1",
-    "eslint": "^8",
-    "eslint-config-next": "14.1.0",
-    "postcss": "^8",
-    "tailwindcss": "^3.3.0",
-    "typescript": "^5"
+    "@types/node": "^22.5.5",
+    "@types/react": "^18.3.8",
+    "@types/react-dom": "^18.3.0",
+    "@types/uuid": "^10.0.0",
+    "autoprefixer": "^10.4.20",
+    "eslint": "^8.0.0",
+    "eslint-config-next": "14.2.13",
+    "postcss": "^8.4.47",
+    "tailwindcss": "^3.4.12",
+    "typescript": "^5.6.2"
   }
 }

package.json

@@ -5,11 +5,12 @@ import { getSelectedModel } from "@/lib/model-helper";
 import { ChatOllama } from "@langchain/community/chat_models/ollama";
 import { AIMessage, HumanMessage } from "@langchain/core/messages";
 import { BytesOutputParser } from "@langchain/core/output_parsers";
-import { ChatRequestOptions } from "ai";
+import { Attachment, ChatRequestOptions } from "ai";
 import { Message, useChat } from "ai/react";
 import React, { useEffect } from "react";
 import { toast } from "sonner";
 import { v4 as uuidv4 } from "uuid";
+import useChatStore from "../hooks/useChatStore";
 
 export default function Page({ params }: { params: { id: string } }) {
   const {
@@ -40,6 +41,9 @@ export default function Page({ params }: { params: { id: string } }) {
   const [ollama, setOllama] = React.useState<ChatOllama>();
   const env = process.env.NODE_ENV;
   const [loadingSubmit, setLoadingSubmit] = React.useState(false);
+  const formRef = React.useRef<HTMLFormElement>(null);
+  const base64Images = useChatStore((state) => state.base64Images);
+  const setBase64Images = useChatStore((state) => state.setBase64Images);
 
   useEffect(() => {
     if (env === "production") {
@@ -120,21 +124,36 @@ export default function Page({ params }: { params: { id: string } }) {
 
     setMessages([...messages]);
 
+    const attachments: Attachment[] = base64Images
+    ? base64Images.map((image) => ({
+        contentType: 'image/base64', // Content type for base64 images
+        url: image, // The base64 image data
+      }))
+    : [];
+
     // Prepare the options object with additional body data, to pass the model.
     const requestOptions: ChatRequestOptions = {
       options: {
         body: {
           selectedModel: selectedModel,
         },
       },
+      ...(base64Images && {
+        data: {
+          images: base64Images,
+        },
+        experimental_attachments: attachments
+      }),
     };
 
     if (env === "production" && selectedModel !== "REST API") {
       handleSubmitProduction(e);
+      setBase64Images(null)
     } else {
       // use the /api/chat route
       // Call the handleSubmit function with the options
       handleSubmit(e, requestOptions);
+      setBase64Images(null)
     }
   };
 
@@ -162,6 +181,7 @@ export default function Page({ params }: { params: { id: string } }) {
         stop={stop}
         navCollapsedSize={10}
         defaultLayout={[30, 160]}
+        formRef={formRef}
         setMessages={setMessages}
         setInput={setInput}
       />

src/app/[id]/page.tsx

@@ -1,31 +1,35 @@
-import { StreamingTextResponse, Message } from "ai";
-import { ChatOllama } from "@langchain/community/chat_models/ollama";
-import { AIMessage, HumanMessage } from "@langchain/core/messages";
-import { BytesOutputParser } from "@langchain/core/output_parsers";
+import { createOllama } from 'ollama-ai-provider';
+import { streamText, convertToCoreMessages, CoreMessage, UserContent } from 'ai';
 
 export const runtime = "edge";
 export const dynamic = "force-dynamic";
 
 export async function POST(req: Request) {
-  const { messages, selectedModel } = await req.json();
+  // Destructure request data
+  const { messages, selectedModel, data } = await req.json();
 
-  const model = new ChatOllama({
-    baseUrl: process.env.NEXT_PUBLIC_OLLAMA_URL || "http://localhost:11434",
-    model: selectedModel,
-  });
+  const initialMessages = messages.slice(0, -1); 
+  const currentMessage = messages[messages.length - 1]; 
+
+  const ollama = createOllama({});
 
-  const parser = new BytesOutputParser();
+  // Build message content array directly
+  const messageContent: UserContent = [{ type: 'text', text: currentMessage.content }];
 
-  const stream = await model
-    .pipe(parser)
-    .stream(
-      (messages as Message[]).map((m) =>
-        m.role == "user"
-          ? new HumanMessage(m.content)
-          : new AIMessage(m.content)
-      )
-    );
+  // Add images if they exist
+  data?.images?.forEach((imageUrl: string) => {
+    const image = new URL(imageUrl);
+    messageContent.push({ type: 'image', image });
+  });
 
+  // Stream text using the ollama model
+  const result = await streamText({
+    model: ollama(selectedModel),
+    messages: [
+      ...convertToCoreMessages(initialMessages),
+      { role: 'user', content: messageContent },
+    ],
+  });
 
-  return new StreamingTextResponse(stream);
+  return result.toDataStreamResponse();
 }

src/app/api/chat/route.ts

@@ -0,0 +1,29 @@
+
+import { CoreMessage } from "ai";
+import { create } from "zustand";
+
+interface State {
+  base64Images: string[] | null;
+  messages: CoreMessage[];
+}
+
+interface Actions {
+  setBase64Images: (base64Images: string[] | null) => void;
+  setMessages: (
+    fn: (
+      messages: CoreMessage[]
+    ) => CoreMessage[]
+  ) => void;
+}
+
+const useChatStore = create<State & Actions>()(
+  (set) => ({
+    base64Images: null,
+    setBase64Images: (base64Images) => set({ base64Images }),
+
+    messages: [],
+    setMessages: (fn) => set((state) => ({ messages: fn(state.messages) })),
+  })
+)
+
+export default useChatStore;

src/app/hooks/useChatStore.ts

@@ -15,11 +15,12 @@ import { getSelectedModel } from "@/lib/model-helper";
 import { ChatOllama } from "@langchain/community/chat_models/ollama";
 import { AIMessage, HumanMessage } from "@langchain/core/messages";
 import { BytesOutputParser } from "@langchain/core/output_parsers";
-import { ChatRequestOptions } from "ai";
+import { Attachment, ChatRequestOptions } from "ai";
 import { Message, useChat } from "ai/react";
 import React, { useEffect, useRef, useState } from "react";
 import { toast } from "sonner";
 import { v4 as uuidv4 } from "uuid";
+import useChatStore from "./hooks/useChatStore";
 
 export default function Home() {
   const {
@@ -29,6 +30,7 @@ export default function Home() {
     handleSubmit,
     isLoading,
     error,
+    data,
     stop,
     setMessages,
     setInput,
@@ -52,6 +54,8 @@ export default function Home() {
   const env = process.env.NODE_ENV;
   const [loadingSubmit, setLoadingSubmit] = React.useState(false);
   const formRef = useRef<HTMLFormElement>(null);
+  const base64Images = useChatStore((state) => state.base64Images);
+  const setBase64Images = useChatStore((state) => state.setBase64Images);
 
   useEffect(() => {
     if (messages.length < 1) {
@@ -85,7 +89,7 @@ export default function Home() {
     }
   }, [selectedModel]);
 
-  const addMessage = (Message: any) => {
+  const addMessage = (Message: Message) => {
     messages.push(Message);
     window.dispatchEvent(new Event("storage"));
     setMessages([...messages]);
@@ -145,20 +149,38 @@ export default function Home() {
 
     setMessages([...messages]);
 
+    const attachments: Attachment[] = base64Images
+    ? base64Images.map((image) => ({
+        contentType: 'image/base64', // Content type for base64 images
+        url: image, // The base64 image data
+      }))
+    : [];
+
     // Prepare the options object with additional body data, to pass the model.
     const requestOptions: ChatRequestOptions = {
       options: {
         body: {
           selectedModel: selectedModel,
         },
       },
+      ...(base64Images && {
+        data: {
+          images: base64Images,
+        },
+        experimental_attachments: attachments
+      }),
     };
 
+    messages.slice(0, -1)
+    
+
     if (env === "production") {
       handleSubmitProduction(e);
+      setBase64Images(null)
     } else {
       // Call the handleSubmit function with the options
       handleSubmit(e, requestOptions);
+      setBase64Images(null)
     }
   };