feat: Adds support for audio file uploads

Allows users to upload and process audio files (mp3, wav)

This commit introduces the ability to upload audio files, process them, and include them in chat messages.
It reads audio files as base64 data and includes the data and format in the API payload.

Author: allozaur

tools/server/webui/src/lib/services/chat.ts

@@ -282,6 +282,7 @@ export class ChatService {
 
 		// Add image files
 		const imageFiles = message.extra.filter((extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile => extra.type === 'imageFile');
+
 		for (const image of imageFiles) {
 			contentParts.push({
 				type: 'image_url',
@@ -291,15 +292,30 @@ export class ChatService {
 
 		// Add text files as additional text content
 		const textFiles = message.extra.filter((extra: DatabaseMessageExtra): extra is DatabaseMessageExtraTextFile => extra.type === 'textFile');
+
 		for (const textFile of textFiles) {
 			contentParts.push({
 				type: 'text',
 				text: `\n\n--- File: ${textFile.name} ---\n${textFile.content}`
 			});
 		}
 
+		// Add audio files
+		const audioFiles = message.extra.filter((extra: DatabaseMessageExtra): extra is DatabaseMessageExtraAudioFile => extra.type === 'audioFile');
+
+		for (const audio of audioFiles) {
+			contentParts.push({
+				type: 'input_audio',
+				input_audio: {
+					data: audio.base64Data,
+					format: audio.mimeType.includes('wav') ? 'wav' : 'mp3'
+				}
+			});
+		}
+
 		// Add PDF files as text content
 		const pdfFiles = message.extra.filter((extra: DatabaseMessageExtra): extra is DatabaseMessageExtraPdfFile => extra.type === 'pdfFile');
+
 		for (const pdfFile of pdfFiles) {
 			if (pdfFile.processedAsImages && pdfFile.images) {
 				// If PDF was processed as images, add each page as an image

tools/server/webui/src/lib/types/api.d.ts

@@ -1,9 +1,13 @@
 export interface ApiChatMessageContentPart {
-	type: 'text' | 'image_url';
+	type: 'text' | 'image_url' | 'input_audio';
 	text?: string;
 	image_url?: {
 		url: string;
 	};
+	input_audio?: {
+		data: string;
+		format: 'wav' | 'mp3';
+	};
 }
 
 export interface ApiChatMessageData {

tools/server/webui/src/lib/utils/convert-files-to-extra.ts

@@ -4,6 +4,27 @@ import { isWebpMimeType, webpBase64UrlToPngDataURL } from "./webp-to-png";
 import { config } from '$lib/stores/settings.svelte';
 import { isLikelyTextFile, readFileAsText } from "./text-files";
 
+function readFileAsBase64(file: File): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const reader = new FileReader();
+
+        reader.onload = () => {
+            // Extract base64 data without the data URL prefix
+            const dataUrl = reader.result as string;
+            const base64 = dataUrl.split(',')[1];
+            resolve(base64);
+        };
+
+        reader.onerror = () => reject(reader.error);
+
+        reader.readAsDataURL(file);
+    });
+}
+
+function isAudioMimeType(mimeType: string): boolean {
+    return mimeType === 'audio/mpeg' || mimeType === 'audio/wav' || mimeType === 'audio/mp3';
+}
+
 export async function parseFilesToMessageExtras(
     files: ChatUploadedFile[]
 ): Promise<DatabaseMessageExtra[]> {
@@ -40,41 +61,64 @@ export async function parseFilesToMessageExtras(
                     base64Url
                 });
             }
+        } else if (isAudioMimeType(file.type)) {
+            // Process audio files (MP3 and WAV)
+            try {
+                const base64Data = await readFileAsBase64(file.file);
+
+                extras.push({
+                    type: 'audioFile',
+                    name: file.name,
+                    base64Data: base64Data,
+                    mimeType: file.type
+                });
+            } catch (error) {
+                console.error(`Failed to process audio file ${file.name}:`, error);
+            }
         } else if (isPdfMimeType(file.type)) {
             try {
+                // Always get base64 data for preview functionality
+                const base64Data = await readFileAsBase64(file.file);
                 const currentConfig = config();
                 const shouldProcessAsImages = Boolean(currentConfig.pdfAsImage);
 
                 if (shouldProcessAsImages) {
                     // Process PDF as images
                     try {
                         const images = await convertPDFToImage(file.file);
+
                         extras.push({
                             type: 'pdfFile',
                             name: file.name,
                             content: `PDF file with ${images.length} pages`,
                             images: images,
-                            processedAsImages: true
+                            processedAsImages: true,
+                            base64Data: base64Data
                         });
                     } catch (imageError) {
                         console.warn(`Failed to process PDF ${file.name} as images, falling back to text:`, imageError);
+
                         // Fallback to text processing
                         const content = await convertPDFToText(file.file);
+
                         extras.push({
                             type: 'pdfFile',
                             name: file.name,
                             content: content,
-                            processedAsImages: false
+                            processedAsImages: false,
+                            base64Data: base64Data
                         });
                     }
                 } else {
                     // Process PDF as text (default)
                     const content = await convertPDFToText(file.file);
+
                     extras.push({
                         type: 'pdfFile',
                         name: file.name,
                         content: content,
-                        processedAsImages: false
+                        processedAsImages: false,
+                        base64Data: base64Data
                     });
                 }
             } catch (error) {

tools/server/webui/src/lib/utils/process-uploaded-files.ts

@@ -65,8 +65,12 @@ export async function processFilesToChatUploaded(files: File[]): Promise<ChatUpl
 			} else if (isPdfMimeType(file.type)) {
 				// PDFs handled later when building extras; keep metadata only
 				results.push(base);
+			} else if (file.type.startsWith('audio/')) {
+				// Generate preview URL for audio files
+				const preview = await readFileAsDataURL(file);
+				results.push({ ...base, preview });
 			} else {
-				// Other files: add as-is (audio, etc.)
+				// Other files: add as-is
 				results.push(base);
 			}
 		} catch (error) {