From f4af3f3a894e8a654f6e8dae681596446e4853a5 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 7 May 2025 19:16:43 +0200 Subject: [PATCH 01/22] rework the input area --- tools/server/webui/package-lock.json | 93 +++++++++- tools/server/webui/package.json | 2 + tools/server/webui/src/App.tsx | 2 + .../webui/src/components/ChatMessage.tsx | 2 +- .../webui/src/components/ChatScreen.tsx | 166 +++++++++++++----- tools/server/webui/src/components/Header.tsx | 54 ++---- tools/server/webui/src/components/Sidebar.tsx | 15 +- .../src/components/useChatExtraContext.tsx | 80 +++++++++ tools/server/webui/src/utils/llama-vscode.ts | 29 ++- tools/server/webui/src/utils/types.ts | 8 +- 10 files changed, 334 insertions(+), 117 deletions(-) create mode 100644 tools/server/webui/src/components/useChatExtraContext.tsx diff --git a/tools/server/webui/package-lock.json b/tools/server/webui/package-lock.json index b2e3cf94aca41..2c23a7580b38c 100644 --- a/tools/server/webui/package-lock.json +++ b/tools/server/webui/package-lock.json @@ -21,6 +21,8 @@ "postcss": "^8.4.49", "react": "^18.3.1", "react-dom": "^18.3.1", + "react-dropzone": "^14.3.8", + "react-hot-toast": "^2.5.2", "react-markdown": "^9.0.3", "react-router": "^7.1.5", "rehype-highlight": "^7.0.2", @@ -2058,6 +2060,15 @@ "dev": true, "license": "Python-2.0" }, + "node_modules/attr-accept": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.5.tgz", + "integrity": "sha512-0bDNnY/u6pPwHDMoF0FieU354oBi0a8rD9FcsLwzcGWbc8KS8KPIi7y+s13OlVY+gMWc/9xEMUgNE6Qm8ZllYQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/autoprefixer": { "version": "10.4.20", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.20.tgz", @@ -2804,6 +2815,18 @@ "node": ">=16.0.0" } }, + "node_modules/file-selector": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-2.1.2.tgz", + "integrity": "sha512-QgXo+mXTe8ljeqUFaX3QVHc5osSItJ/Km+xpocx0aSqWGMSCf6qYs/VnzZgS864Pjn5iceMRFigeAV7AfTlaig==", + "license": "MIT", + "dependencies": { + "tslib": "^2.7.0" + }, + "engines": { + "node": ">= 12" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -2917,6 +2940,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/goober": { + "version": "2.1.16", + "resolved": "https://registry.npmjs.org/goober/-/goober-2.1.16.tgz", + "integrity": "sha512-erjk19y1U33+XAMe1VTvIONHYoSqE4iS7BYUZfHaqeohLmnC0FdxEh7rQU+6MZ4OajItzjZFSRtVANrQwNq6/g==", + "license": "MIT", + "peerDependencies": { + "csstype": "^3.0.10" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -4674,6 +4706,15 @@ "node": ">=0.10.0" } }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -4872,6 +4913,17 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, "node_modules/property-information": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.5.0.tgz", @@ -4938,6 +4990,46 @@ "react": "^18.3.1" } }, + "node_modules/react-dropzone": { + "version": "14.3.8", + "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.3.8.tgz", + "integrity": "sha512-sBgODnq+lcA4P296DY4wacOZz3JFpD99fp+hb//iBO2HHnyeZU3FwWyXJ6salNpqQdsZrgMrotuko/BdJMV8Ug==", + "license": "MIT", + "dependencies": { + "attr-accept": "^2.2.4", + "file-selector": "^2.1.0", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">= 10.13" + }, + "peerDependencies": { + "react": ">= 16.8 || 18.0.0" + } + }, + "node_modules/react-hot-toast": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/react-hot-toast/-/react-hot-toast-2.5.2.tgz", + "integrity": "sha512-Tun3BbCxzmXXM7C+NI4qiv6lT0uwGh4oAfeJyNOjYUejTsm35mK9iCaYLGv8cBz9L5YxZLx/2ii7zsIwPtPUdw==", + "license": "MIT", + "dependencies": { + "csstype": "^3.1.3", + "goober": "^2.1.16" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "react": ">=16", + "react-dom": ">=16" + } + }, + "node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, "node_modules/react-markdown": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-9.0.3.tgz", @@ -5814,7 +5906,6 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "devOptional": true, "license": "0BSD" }, "node_modules/turbo-stream": { diff --git a/tools/server/webui/package.json b/tools/server/webui/package.json index 6ac06b1a49bd3..ab1b920bdc5d6 100644 --- a/tools/server/webui/package.json +++ b/tools/server/webui/package.json @@ -24,6 +24,8 @@ "postcss": "^8.4.49", "react": "^18.3.1", "react-dom": "^18.3.1", + "react-dropzone": "^14.3.8", + "react-hot-toast": "^2.5.2", "react-markdown": "^9.0.3", "react-router": "^7.1.5", "rehype-highlight": "^7.0.2", diff --git a/tools/server/webui/src/App.tsx b/tools/server/webui/src/App.tsx index cc4659e1529fe..3b00a8f909ad6 100644 --- a/tools/server/webui/src/App.tsx +++ b/tools/server/webui/src/App.tsx @@ -4,6 +4,7 @@ import Sidebar from './components/Sidebar'; import { AppContextProvider, useAppContext } from './utils/app.context'; import ChatScreen from './components/ChatScreen'; import SettingDialog from './components/SettingDialog'; +import { Toaster } from 'react-hot-toast'; function App() { return ( @@ -40,6 +41,7 @@ function AppLayout() { onClose={() => setShowSettings(false)} /> } + ); } diff --git a/tools/server/webui/src/components/ChatMessage.tsx b/tools/server/webui/src/components/ChatMessage.tsx index 40ea74711f349..c7d14c8c87164 100644 --- a/tools/server/webui/src/components/ChatMessage.tsx +++ b/tools/server/webui/src/components/ChatMessage.tsx @@ -88,7 +88,7 @@ export default function ChatMessage({
{/* textarea for editing message */} diff --git a/tools/server/webui/src/components/ChatScreen.tsx b/tools/server/webui/src/components/ChatScreen.tsx index a2e3ee9975834..edc4c1aec3ef4 100644 --- a/tools/server/webui/src/components/ChatScreen.tsx +++ b/tools/server/webui/src/components/ChatScreen.tsx @@ -1,12 +1,23 @@ import { useEffect, useMemo, useState } from 'react'; import { CallbackGeneratedChunk, useAppContext } from '../utils/app.context'; import ChatMessage from './ChatMessage'; -import { CanvasType, Message, PendingMessage } from '../utils/types'; +import { CanvasType, Message, MessageExtraContext, PendingMessage } from '../utils/types'; import { classNames, cleanCurrentUrl, throttle } from '../utils/misc'; import CanvasPyInterpreter from './CanvasPyInterpreter'; import StorageUtils from '../utils/storage'; import { useVSCodeContext } from '../utils/llama-vscode'; import { useChatTextarea, ChatTextareaApi } from './useChatTextarea.ts'; +import { + ArrowUpIcon, + StopIcon, + PaperClipIcon, + DocumentTextIcon, +} from '@heroicons/react/24/solid'; +import { + ChatExtraContextApi, + useChatExtraContext, +} from './useChatExtraContext.tsx'; +import Dropzone from 'react-dropzone'; /** * A message display is a message node with additional information for rendering. @@ -102,10 +113,8 @@ export default function ChatScreen() { } = useAppContext(); const textarea: ChatTextareaApi = useChatTextarea(prefilledMsg.content()); - - const { extraContext, clearExtraContext } = useVSCodeContext(textarea); - // TODO: improve this when we have "upload file" feature - const currExtra: Message['extra'] = extraContext ? [extraContext] : undefined; + const extraContext = useChatExtraContext(); + useVSCodeContext(textarea, extraContext); // keep track of leaf node for rendering const [currNodeId, setCurrNodeId] = useState(-1); @@ -146,7 +155,7 @@ export default function ChatScreen() { currConvId, lastMsgNodeId, lastInpMsg, - currExtra, + extraContext.items, onChunk )) ) { @@ -154,7 +163,7 @@ export default function ChatScreen() { textarea.setValue(lastInpMsg); } // OK - clearExtraContext(); + extraContext.clearItems(); }; // for vscode context @@ -253,41 +262,13 @@ export default function ChatScreen() {
{/* chat input */} -
- - - {isGenerating(currConvId ?? '') ? ( - - ) : ( - - )} -
+ stopGenerating(currConvId ?? '')} + isGenerating={isGenerating(currConvId ?? '')} + />
{canvasData?.type === CanvasType.PY_INTERPRETER && ( @@ -297,3 +278,104 @@ export default function ChatScreen() {
); } + +function ChatInput({ + textarea, + extraContext, + onSend, + onStop, + isGenerating, +}: { + textarea: ChatTextareaApi; + extraContext: ChatExtraContextApi; + onSend: () => void; + onStop: () => void; + isGenerating: boolean; +}) { + const [isDrag, setIsDrag] = useState(false); + + return ( +
+ { + setIsDrag(false); + extraContext.onFileAdded(files); + }} + onDragEnter={() => setIsDrag(true)} + onDragLeave={() => setIsDrag(false)} + multiple={true} + > + {({ getRootProps, getInputProps }) => ( +
+ + + {/* buttons area */} +
+ + + {isGenerating ? ( + + ) : ( + + )} +
+
+ )} + +
+ ); +} + +function ChatInputExtraContextItem({}: { + idx: number, + item: MessageExtraContext, +}) { + +} diff --git a/tools/server/webui/src/components/Header.tsx b/tools/server/webui/src/components/Header.tsx index 4c6b291e61bcb..5fb46d8f80e9f 100644 --- a/tools/server/webui/src/components/Header.tsx +++ b/tools/server/webui/src/components/Header.tsx @@ -5,6 +5,12 @@ import { classNames } from '../utils/misc'; import daisyuiThemes from 'daisyui/theme/object'; import { THEMES } from '../Config'; import { useNavigate } from 'react-router'; +import { + Cog8ToothIcon, + SunIcon, + EllipsisVerticalIcon, + Bars3Icon, +} from '@heroicons/react/24/outline'; export default function Header() { const navigate = useNavigate(); @@ -55,19 +61,7 @@ export default function Header() {
{/* open sidebar button */}
llama.cpp
@@ -83,16 +77,7 @@ export default function Header() { className="btn m-1" disabled={isCurrConvGenerating} > - - - + {/* dropdown menu */}
@@ -130,16 +105,7 @@ export default function Header() {
- - - +
    - - - +
diff --git a/tools/server/webui/src/components/useChatExtraContext.tsx b/tools/server/webui/src/components/useChatExtraContext.tsx new file mode 100644 index 0000000000000..024e7be6eb9c6 --- /dev/null +++ b/tools/server/webui/src/components/useChatExtraContext.tsx @@ -0,0 +1,80 @@ +import { useState } from 'react'; +import { MessageExtra } from '../utils/types'; +import toast from 'react-hot-toast'; + +// Interface describing the API returned by the hook +export interface ChatExtraContextApi { + items?: MessageExtra[]; // undefined if empty, similar to Message['extra'] + addItems: (items: MessageExtra[]) => void; + removeItem: (idx: number) => void; + clearItems: () => void; + onFileAdded: (files: File[]) => void; // used by "upload" button +} + +export function useChatExtraContext(): ChatExtraContextApi { + const [items, setItems] = useState([]); + + const addItems = (newItems: MessageExtra[]) => { + setItems((prev) => [...prev, ...newItems]); + }; + + const removeItem = (idx: number) => { + setItems((prev) => prev.filter((_, i) => i !== idx)); + }; + + const clearItems = () => { + setItems([]); + }; + + const onFileAdded = (files: File[]) => { + for (const file of files) { + const mimeType = file.type; + console.debug({ mimeType, file }); + if (file.size > 10 * 1024 * 1024) { + toast.error('File is too large. Maximum size is 10MB.'); + break; + } + if (mimeType.startsWith('text/')) { + const reader = new FileReader(); + reader.onload = (event) => { + if (event.target?.result) { + addItems([ + { + type: 'textFile', + name: file.name, + content: event.target.result as string, + }, + ]); + } + }; + reader.readAsText(file); + } else if (mimeType.startsWith('image/')) { + // TODO @ngxson : throw an error if the server does not support image input + const reader = new FileReader(); + reader.onload = (event) => { + if (event.target?.result) { + addItems([ + { + type: 'imageFile', + name: file.name, + base64Url: event.target.result as string, + }, + ]); + } + }; + reader.readAsDataURL(file); + } else { + // TODO @ngxson : support all other file formats like .pdf, .py, .bat, .c, etc + toast.error('Unsupported file type.'); + } + } + }; + + return { + items: items.length > 0 ? items : undefined, + addItems, + removeItem, + clearItems, + onFileAdded, + }; +} diff --git a/tools/server/webui/src/utils/llama-vscode.ts b/tools/server/webui/src/utils/llama-vscode.ts index 55ebdcffc420e..dac89f1e34618 100644 --- a/tools/server/webui/src/utils/llama-vscode.ts +++ b/tools/server/webui/src/utils/llama-vscode.ts @@ -1,6 +1,7 @@ import { useEffect, useState } from 'react'; import { MessageExtraContext } from './types'; import { ChatTextareaApi } from '../components/useChatTextarea.ts'; +import { ChatExtraContextApi } from '../components/useChatExtraContext.tsx'; // Extra context when using llama.cpp WebUI from llama-vscode, inside an iframe // Ref: https://github.com/ggml-org/llama.cpp/pull/11940 @@ -15,11 +16,10 @@ interface SetTextEvData { * window.postMessage({ command: 'setText', text: 'Spot the syntax error', context: 'def test()\n return 123' }, '*'); */ -export const useVSCodeContext = (textarea: ChatTextareaApi) => { - const [extraContext, setExtraContext] = useState( - null - ); - +export const useVSCodeContext = ( + textarea: ChatTextareaApi, + extraContext: ChatExtraContextApi +) => { // Accept setText message from a parent window and set inputMsg and extraContext useEffect(() => { const handleMessage = (event: MessageEvent) => { @@ -27,10 +27,13 @@ export const useVSCodeContext = (textarea: ChatTextareaApi) => { const data: SetTextEvData = event.data; textarea.setValue(data?.text); if (data?.context && data.context.length > 0) { - setExtraContext({ - type: 'context', - content: data.context, - }); + extraContext.clearItems(); + extraContext.addItems([ + { + type: 'context', + content: data.context, + }, + ]); } textarea.focus(); setTimeout(() => { @@ -41,7 +44,7 @@ export const useVSCodeContext = (textarea: ChatTextareaApi) => { window.addEventListener('message', handleMessage); return () => window.removeEventListener('message', handleMessage); - }, [textarea]); + }, [textarea, extraContext]); // Add a keydown listener that sends the "escapePressed" message to the parent window useEffect(() => { @@ -55,9 +58,5 @@ export const useVSCodeContext = (textarea: ChatTextareaApi) => { return () => window.removeEventListener('keydown', handleKeyDown); }, []); - return { - extraContext, - // call once the user message is sent, to clear the extra context - clearExtraContext: () => setExtraContext(null), - }; + return {}; }; diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index 0eb774001ecc5..88e776cde48ed 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -48,7 +48,7 @@ export interface Message { children: Message['id'][]; } -type MessageExtra = MessageExtraTextFile | MessageExtraContext; // TODO: will add more in the future +export type MessageExtra = MessageExtraTextFile | MessageExtraImageFile | MessageExtraContext; export interface MessageExtraTextFile { type: 'textFile'; @@ -56,6 +56,12 @@ export interface MessageExtraTextFile { content: string; } +export interface MessageExtraImageFile { + type: 'imageFile'; + name: string; + base64Url: string; +} + export interface MessageExtraContext { type: 'context'; content: string; From 7d59402924c468a17e425cd2478f7ea13dd4e537 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 7 May 2025 22:18:16 +0200 Subject: [PATCH 02/22] process selected file --- common/chat.cpp | 4 +- .../components/ChatInputExtraContextItem.tsx | 92 +++++++++++++++++++ .../webui/src/components/ChatMessage.tsx | 33 +------ .../webui/src/components/ChatScreen.tsx | 28 ++++-- tools/server/webui/src/components/Sidebar.tsx | 4 +- tools/server/webui/src/utils/app.context.tsx | 3 +- tools/server/webui/src/utils/llama-vscode.ts | 1 + tools/server/webui/src/utils/misc.ts | 47 ++++++++-- tools/server/webui/src/utils/types.ts | 21 ++++- 9 files changed, 181 insertions(+), 52 deletions(-) create mode 100644 tools/server/webui/src/components/ChatInputExtraContextItem.tsx diff --git a/common/chat.cpp b/common/chat.cpp index bbc5f087cdcc0..ad3d4aa99a926 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -125,7 +125,9 @@ std::vector common_chat_msgs_parse_oaicompat(const json & messa msgs.push_back(msg); } } catch (const std::exception & e) { - throw std::runtime_error("Failed to parse messages: " + std::string(e.what()) + "; messages = " + messages.dump(2)); + // @ngxson : disable otherwise it's bloating the API response + // printf("%s\n", std::string("; messages = ") + messages.dump(2)); + throw std::runtime_error("Failed to parse messages: " + std::string(e.what())); } return msgs; diff --git a/tools/server/webui/src/components/ChatInputExtraContextItem.tsx b/tools/server/webui/src/components/ChatInputExtraContextItem.tsx new file mode 100644 index 0000000000000..93b281dad6dbf --- /dev/null +++ b/tools/server/webui/src/components/ChatInputExtraContextItem.tsx @@ -0,0 +1,92 @@ +import { DocumentTextIcon, XMarkIcon } from '@heroicons/react/24/outline'; +import { MessageExtra } from '../utils/types'; +import { useState } from 'react'; +import { classNames } from '../utils/misc'; + +export default function ChatInputExtraContextItem({ + items, + removeItem, + clickToShow, +}: { + items?: MessageExtra[]; + removeItem?: (index: number) => void; + clickToShow?: boolean; +}) { + const [show, setShow] = useState(-1); + const showingItem = show >= 0 ? items?.[show] : undefined; + + if (!items) return null; + + return ( +
+ {items.map((item, i) => ( +
clickToShow && setShow(i)} + > + {removeItem && ( +
+ +
+ )} + +
+ {item.type === 'imageFile' ? ( + <> + {item.name} + + ) : ( + <> +
+ +
+ +
+ {item.name} +
+ + )} +
+
+ ))} + + {showingItem && ( + +
+
+ {showingItem.name} + +
+ {showingItem.type === 'imageFile' ? ( + {showingItem.name} + ) : ( +
+ 
+                  {showingItem.content}
+
+
+ )} +
+
setShow(-1)}>
+ + )} +
+ ); +} diff --git a/tools/server/webui/src/components/ChatMessage.tsx b/tools/server/webui/src/components/ChatMessage.tsx index c7d14c8c87164..84c4ba8e1d8ef 100644 --- a/tools/server/webui/src/components/ChatMessage.tsx +++ b/tools/server/webui/src/components/ChatMessage.tsx @@ -4,6 +4,7 @@ import { Message, PendingMessage } from '../utils/types'; import { classNames } from '../utils/misc'; import MarkdownDisplay, { CopyButton } from './MarkdownDisplay'; import { ChevronLeftIcon, ChevronRightIcon } from '@heroicons/react/24/outline'; +import ChatInputExtraContextItem from './ChatInputExtraContextItem'; interface SplitMessage { content: PendingMessage['content']; @@ -85,6 +86,10 @@ export default function ChatMessage({ 'chat-end': msg.role === 'user', })} > + {msg.extra && msg.extra.length > 0 && ( + + )} +
)} - {msg.extra && msg.extra.length > 0 && ( -
- - Extra content - -
- {msg.extra.map( - (extra, i) => - extra.type === 'textFile' ? ( -
- {extra.name} - 
{extra.content}
-
- ) : extra.type === 'context' ? ( -
- 
{extra.content}
-
- ) : null // TODO: support other extra types - )} -
-
- )} - { const lastInpMsg = textarea.value(); - if (lastInpMsg.trim().length === 0 || isGenerating(currConvId ?? '')) + if (lastInpMsg.trim().length === 0 || isGenerating(currConvId ?? '')) { + toast.error('Please enter a message'); return; + } textarea.setValue(''); scrollToBottom(false); setCurrNodeId(-1); @@ -312,7 +317,16 @@ function ChatInput({ multiple={true} > {({ getRootProps, getInputProps }) => ( -
+
+ + +