feat: Adjusting types & request for handling the files WIP

Author: allozaur

tools/server/webui/src/lib/services/chat.ts

@@ -1,14 +1,18 @@
-import type { Message } from '$lib/types/database';
-
-export interface LlamaCppServerProps {
-	build_info: string;
-	model_path: string;
-	n_ctx: number;
-	modalities?: {
-		vision: boolean;
-		audio: boolean;
-	};
-}
+import type {
+	ApiChatMessageContentPart,
+	ApiChatMessageData,
+	ApiChatCompletionRequest,
+	ApiChatCompletionStreamChunk,
+	ApiChatCompletionResponse,
+	ApiLlamaCppServerProps
+} from '$lib/types/api';
+
+import type {
+	DatabaseMessage,
+	DatabaseMessageExtra,
+	DatabaseMessageExtraImageFile,
+	DatabaseMessageExtraTextFile
+} from '$lib/types/database'
 
 export class ChatService {
 	private baseUrl: string;
@@ -19,7 +23,7 @@ export class ChatService {
 	}
 
 	async sendMessage(
-		messages: ChatMessageData[],
+		messages: ApiChatMessageData[],
 		options: {
 			stream?: boolean;
 			temperature?: number;
@@ -35,7 +39,7 @@ export class ChatService {
 		this.abort();
 		this.abortController = new AbortController();
 
-		const requestBody: ChatCompletionRequest = {
+		const requestBody: ApiChatCompletionRequest = {
 			messages: messages.map((msg) => ({
 				role: msg.role,
 				content: msg.content
@@ -119,7 +123,7 @@ export class ChatService {
 						}
 
 						try {
-							const parsed: ChatCompletionStreamChunk = JSON.parse(data);
+							const parsed: ApiChatCompletionStreamChunk = JSON.parse(data);
 							const content = parsed.choices[0]?.delta?.content;
 							if (content) {
 								fullResponse += content;
@@ -161,7 +165,7 @@ export class ChatService {
 		onError?: (error: Error) => void
 	): Promise<string> {
 		try {
-			const data: ChatCompletionResponse = await response.json();
+			const data: ApiChatCompletionResponse = await response.json();
 			const content = data.choices[0]?.message?.content || '';
 
 			onComplete?.(content);
@@ -177,10 +181,57 @@ export class ChatService {
 	}
 
 	/**
-	 * Unified method to send chat completions supporting both ChatMessageData and Message types
+	 * Convert Message with extras to ApiChatMessageData format for API requests
+	 */
+	private static convertMessageToChatData(message: DatabaseMessage & { extra?: DatabaseMessageExtra[] }): ApiChatMessageData {
+		// If no extras, return simple text message
+		if (!message.extra || message.extra.length === 0) {
+			return {
+				role: message.role as 'user' | 'assistant' | 'system',
+				content: message.content
+			};
+		}
+
+		// Build multimodal content array
+		const contentParts: ApiChatMessageContentPart[] = [];
+
+		// Add text content first
+		if (message.content) {
+			contentParts.push({
+				type: 'text',
+				text: message.content
+			});
+		}
+
+		// Add image files
+		const imageFiles = message.extra.filter((extra): extra is DatabaseMessageExtraImageFile => extra.type === 'imageFile');
+		imageFiles.forEach((image) => {
+			contentParts.push({
+				type: 'image_url',
+				image_url: { url: image.base64Url }
+			});
+		});
+
+		// Add text files as additional text content
+		const textFiles = message.extra.filter((extra): extra is DatabaseMessageExtraTextFile => extra.type === 'textFile');
+		textFiles.forEach((textFile) => {
+			contentParts.push({
+				type: 'text',
+				text: `\n\n--- File: ${textFile.name} ---\n${textFile.content}`
+			});
+		});
+
+		return {
+			role: message.role as 'user' | 'assistant' | 'system',
+			content: contentParts
+		};
+	}
+
+	/**
+	 * Unified method to send chat completions supporting both ApiChatMessageData and Message types
 	 */
 	async sendChatCompletion(
-		messages: ChatMessageData[] | Message[],
+		messages: (ApiChatMessageData[] | DatabaseMessage[]) | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[],
 		options: {
 			stream?: boolean;
 			temperature?: number;
@@ -190,12 +241,16 @@ export class ChatService {
 			onError?: (error: Error) => void;
 		} = {}
 	): Promise<string | void> {
-		// Normalize messages to ChatMessageData format
-		const normalizedMessages: ChatMessageData[] = messages.map((msg) => ({
-			role: msg.role as ChatMessageData['role'],
-			content: msg.content,
-			timestamp: 'timestamp' in msg ? msg.timestamp : undefined
-		}));
+		// Handle both array formats and convert messages with extras
+		const normalizedMessages: ApiChatMessageData[] = messages.map((msg) => {
+			// Check if this is already a ApiChatMessageData object
+			if ('content' in msg && (typeof msg.content === 'string' || Array.isArray(msg.content))) {
+				return msg as ApiChatMessageData;
+			}
+			
+			// Convert DatabaseMessage with extras to ApiChatMessageData
+			return ChatService.convertMessageToChatData(msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] });
+		});
 
 		// Set default options for API compatibility
 		const finalOptions = {
@@ -212,7 +267,7 @@ export class ChatService {
 	 * Static method for backward compatibility with ApiService
 	 */
 	static async sendChatCompletion(
-		messages: Message[],
+		messages: DatabaseMessage[],
 		onChunk?: (content: string) => void,
 		onComplete?: () => void,
 		onError?: (error: Error) => void
@@ -232,7 +287,7 @@ export class ChatService {
 	/**
 	 * Get server properties - static method for API compatibility
 	 */
-	static async getServerProps(): Promise<LlamaCppServerProps> {
+	static async getServerProps(): Promise<ApiLlamaCppServerProps> {
 		try {
 			const response = await fetch(`${import.meta.env.VITE_BASE_URL}/props`, {
 				headers: {

tools/server/webui/src/lib/types/api.d.ts

@@ -0,0 +1,55 @@
+export interface ApiChatMessageContentPart {
+	type: 'text' | 'image_url';
+	text?: string;
+	image_url?: {
+		url: string;
+	};
+}
+
+export interface ApiChatMessageData {
+	role: ChatRole;
+	content: string | ApiChatMessageContentPart[];
+	timestamp?: number;
+}
+
+export interface ApiLlamaCppServerProps {
+	build_info: string;
+	model_path: string;
+	n_ctx: number;
+	modalities?: {
+		vision: boolean;
+		audio: boolean;
+	};
+}
+
+export interface ApiChatCompletionRequest {
+	messages: Array<{
+		role: ChatRole;
+		content: string | ApiChatMessageContentPart[];
+	}>;
+	stream?: boolean;
+	temperature?: number;
+	max_tokens?: number;
+}
+
+export interface ApiChatCompletionStreamChunk {
+	choices: Array<{
+		delta: {
+			content?: string;
+		};
+	}>;
+	timings?: {
+		prompt_n?: number;
+		prompt_ms?: number;
+		predicted_n?: number;
+		predicted_ms?: number;
+	};
+}
+
+export interface ApiChatCompletionResponse {
+	choices: Array<{
+		message: {
+			content: string;
+		};
+	}>;
+}

tools/server/webui/src/lib/types/chat.d.ts

@@ -1,57 +1,2 @@
-export type ChatRole = 'user' | 'assistant' | 'system';
-
-// todo - currently we're processing <think>...</think> tags from the streamed 'text' response. Should this be done differently?
 export type ChatMessageType = 'root' | 'text' | 'think';
-
-export interface ChatMessageData {
-	role: ChatRole;
-	type: ChatMessageType
-	content: string;
-	thinking?: string; // Content between <think> and </think> tags
-	timestamp?: Date | number;
-}
-
-export interface ChatCompletionRequest {
-	messages: Array<{ role: string; content: string }>;
-	stream?: boolean;
-	temperature?: number;
-	max_tokens?: number;
-	top_p?: number;
-	frequency_penalty?: number;
-	presence_penalty?: number;
-}
-
-export interface ChatCompletionResponse {
-	id: string;
-	object: string;
-	created: number;
-	model: string;
-	choices: Array<{
-		index: number;
-		message: {
-			role: string;
-			content: string;
-		};
-		finish_reason: string;
-	}>;
-	usage?: {
-		prompt_tokens: number;
-		completion_tokens: number;
-		total_tokens: number;
-	};
-}
-
-export interface ChatCompletionStreamChunk {
-	id: string;
-	object: string;
-	created: number;
-	model: string;
-	choices: Array<{
-		index: number;
-		delta: {
-			role?: string;
-			content?: string;
-		};
-		finish_reason?: string;
-	}>;
-}
+export type ChatRole = 'user' | 'assistant' | 'system';

tools/server/webui/src/lib/types/database.d.ts

@@ -1,31 +1,48 @@
-export interface Conversation {
+export interface DatabaseAppSettings {
+	id: string;
+	theme: 'light' | 'dark' | 'system';
+	model: string;
+	temperature: number;
+}
+
+
+export interface DatabaseConversation {
 	currNode: string | null;
 	id: string;
 	lastModified: number;
 	name: string;
 }
 
-export interface Message {
+export interface DatabaseMessageExtraAudioFile {
+	type: 'audioFile';
+	name: string;
+	base64Data: string;
+	mimeType: string;
+}
+
+export interface DatabaseMessageExtraImageFile {
+	type: 'imageFile';
+	name: string;
+	base64Url: string;
+}
+
+export interface DatabaseMessageExtraTextFile {
+	type: 'textFile';
+	name: string;
+	content: string;
+}
+
+export type DatabaseMessageExtra = DatabaseMessageExtraImageFile | DatabaseMessageExtraTextFile | DatabaseMessageExtraAudioFile;
+
+export interface DatabaseMessage {
 	id: string;
 	convId: string;
-	type: 'root' | 'text' | 'think';
+	type: ChatMessageType;
 	timestamp: number;
-	role: 'system' | 'user' | 'assistant';
+	role: ChatRole;
 	content: string;
 	parent: string;
 	thinking: string;
 	children: string[];
-}
-
-export interface DatabaseAppSettings {
-	id: string;
-	theme: 'light' | 'dark' | 'system';
-	model: string;
-	temperature: number;
-	maxTokens: number;
-	topP: number;
-	topK: number;
-	repeatPenalty: number;
-	seed: number;
-	systemPrompt: string;
-}
+	extra?: DatabaseMessageExtra[];
+}