feat: Improves file upload handling with modality support

Refactors file upload logic to handle modality support. It now dynamically filters files based on the capabilities of the active model (vision, audio) and displays a detailed error message when incompatible files are selected, preventing unexpected behavior and improving user experience.

- Updates file input to dynamically accept file types based on model capabilities.
- Introduces an alert dialog to inform users about unsupported file types and reasons, with specific messages for both generally unsupported files and modality-related issues.
- Modifies PDF processing to ensure PDF files are processed as text for non-vision models and images for vision models, adapting the setting when necessary.

Author: allozaur

tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActionButtons.svelte

@@ -25,6 +25,11 @@
 		isRecording = false,
 		class: className = ''
 	}: Props = $props();
+
+	const fileUploadDisabled = disabled || isLoading;
+	const fileUploadTooltipText = !supportsVision() 
+		? 'Text files and PDFs supported. Images, audio, and video require vision models.'
+		: 'Attach files';
 </script>
 
 <div class="flex items-center justify-between gap-1 {className}">
@@ -33,21 +38,17 @@
 			<Tooltip.Trigger>
 				<Button
 					type="button"
-					class="text-muted-foreground bg-transparent hover:bg-foreground/10 hover:text-foreground h-8 w-8 rounded-full p-0 {!supportsVision()
-						? 'opacity-50 cursor-not-allowed'
-						: ''}"
-					disabled={disabled || isLoading || !supportsVision()}
+					class="text-muted-foreground bg-transparent hover:bg-foreground/10 hover:text-foreground h-8 w-8 rounded-full p-0"
+					disabled={fileUploadDisabled}
 					onclick={onFileUpload}
 				>
 					<span class="sr-only">Attach files</span>
 					<Paperclip class="h-4 w-4" />
 				</Button>
 			</Tooltip.Trigger>
-			{#if !supportsVision()}
-				<Tooltip.Content>
-					<p>Current model does not support vision</p>
-				</Tooltip.Content>
-			{/if}
+			<Tooltip.Content>
+				<p>{fileUploadTooltipText}</p>
+			</Tooltip.Content>
 		</Tooltip.Root>
 	</div>
 

tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormFileInputInvisible.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
 	import { ALL_SUPPORTED_EXTENSIONS, ALL_SUPPORTED_MIME_TYPES } from '$lib/constants/supported-file-types';
+	import { generateModalityAwareAcceptString } from '$lib/utils/modality-file-validation';
 
 	interface Props {
 		accept?: string;
@@ -15,12 +16,15 @@
 	].join(',');
 
 	let {
-		accept = defaultAccept,
+		accept,
 		multiple = true,
 		onFileSelect,
 		class: className = ''
 	}: Props = $props();
 
+	// Use modality-aware accept string by default, but allow override
+	const finalAccept = $derived(accept ?? generateModalityAwareAcceptString());
+
 	let fileInputElement: HTMLInputElement | undefined;
 
 	export function click() {
@@ -39,7 +43,7 @@
 	bind:this={fileInputElement}
 	type="file"
 	{multiple}
-	{accept}
+	accept={finalAccept}
 	onchange={handleFileSelect}
 	class="hidden {className}"
 />

tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte

@@ -3,6 +3,7 @@
 	import { processFilesToChatUploaded } from '$lib/utils/process-uploaded-files';
 	import { serverStore } from '$lib/stores/server.svelte';
 	import { isFileTypeSupported } from '$lib/constants/supported-file-types';
+	import { filterFilesByModalities, generateModalityErrorMessage } from '$lib/utils/modality-file-validation';
 	import { ChatForm, ChatScreenHeader, ChatMessages, ServerInfo } from '$lib/components/app';
 	import {
 		activeMessages,
@@ -17,6 +18,7 @@
 	import { AUTO_SCROLL_THRESHOLD } from '$lib/constants/auto-scroll';
 	import { navigating } from '$app/state';
 	import ChatScreenDragOverlay from './ChatScreenDragOverlay.svelte';
+	import * as AlertDialog from '$lib/components/ui/alert-dialog';
 
 	let { showCenteredEmpty = false } = $props();
 	let chatScrollContainer: HTMLDivElement | undefined = $state();
@@ -25,6 +27,20 @@
 	let uploadedFiles = $state<ChatUploadedFile[]>([]);
 	let isDragOver = $state(false);
 	let dragCounter = $state(0);
+	
+	// Alert Dialog state for file upload errors
+	let showFileErrorDialog = $state(false);
+	let fileErrorData = $state<{
+		generallyUnsupported: File[];
+		modalityUnsupported: File[];
+		modalityReasons: Record<string, string>;
+		supportedTypes: string[];
+	}>({
+		generallyUnsupported: [],
+		modalityUnsupported: [],
+		modalityReasons: {},
+		supportedTypes: []
+	});
 
 	const isEmpty = $derived(
 		showCenteredEmpty && !activeConversation() && activeMessages().length === 0 && !isLoading()
@@ -117,24 +133,38 @@
 	}
 
 	async function processFiles(files: File[]) {
-		const supportedFiles: File[] = [];
-		const unsupportedFiles: File[] = [];
+		// First filter by general file type support
+		const generallySupported: File[] = [];
+		const generallyUnsupported: File[] = [];
 
 		for (const file of files) {
 			if (isFileTypeSupported(file.name, file.type)) {
-				supportedFiles.push(file);
+				generallySupported.push(file);
 			} else {
-				unsupportedFiles.push(file);
+				generallyUnsupported.push(file);
 			}
 		}
 
-		if (unsupportedFiles.length > 0) {
-			const fileNames = unsupportedFiles.map(f => f.name).join(', ');
-			const message = unsupportedFiles.length === 1 
-				? `The file "${fileNames}" is not supported. Please upload images (JPG, PNG, GIF, WebP, SVG), audio files (MP3, WAV), PDFs, or text files.`
-				: `The following files are not supported: ${fileNames}. Please upload images (JPG, PNG, GIF, WebP, SVG), audio files (MP3, WAV), PDFs, or text files.`;
+		// Then filter by model modalities
+		const { supportedFiles, unsupportedFiles, modalityReasons } = filterFilesByModalities(generallySupported);
+
+		// Combine all unsupported files
+		const allUnsupportedFiles = [...generallyUnsupported, ...unsupportedFiles];
+
+		if (allUnsupportedFiles.length > 0) {
+			// Determine supported types for current model
+			const supportedTypes: string[] = ['text files', 'PDFs'];
+			if (supportsVision()) supportedTypes.push('images');
+			if (supportsAudio()) supportedTypes.push('audio files');
 			
-			alert(message);
+			// Structure error data for better presentation
+			fileErrorData = {
+				generallyUnsupported,
+				modalityUnsupported: unsupportedFiles,
+				modalityReasons,
+				supportedTypes
+			};
+			showFileErrorDialog = true;
 		}
 
 		if (supportedFiles.length > 0) {
@@ -242,6 +272,69 @@
 	</div>
 {/if}
 
+<!-- File Upload Error Alert Dialog -->
+<AlertDialog.Root bind:open={showFileErrorDialog}>
+	<AlertDialog.Portal>
+		<AlertDialog.Overlay />
+		<AlertDialog.Content class="max-w-md">
+			<AlertDialog.Header>
+				<AlertDialog.Title>File Upload Error</AlertDialog.Title>
+				<AlertDialog.Description class="text-sm text-muted-foreground">
+					Some files cannot be uploaded with the current model.
+				</AlertDialog.Description>
+			</AlertDialog.Header>
+			
+			<div class="space-y-4">
+				<!-- Generally unsupported files -->
+				{#if fileErrorData.generallyUnsupported.length > 0}
+					<div class="space-y-2">
+						<h4 class="text-sm font-medium text-destructive">Unsupported File Types</h4>
+						<div class="space-y-1">
+							{#each fileErrorData.generallyUnsupported as file}
+								<div class="rounded-md bg-destructive/10 px-3 py-2">
+									<p class="text-sm font-mono text-destructive break-all">{file.name}</p>
+									<p class="text-xs text-muted-foreground mt-1">File type not supported</p>
+								</div>
+							{/each}
+						</div>
+					</div>
+				{/if}
+				
+				<!-- Modality-restricted files -->
+				{#if fileErrorData.modalityUnsupported.length > 0}
+					<div class="space-y-2">
+						<h4 class="text-sm font-medium text-destructive">Model Compatibility Issues</h4>
+						<div class="space-y-1">
+							{#each fileErrorData.modalityUnsupported as file}
+								<div class="rounded-md bg-destructive/10 px-3 py-2">
+									<p class="text-sm font-mono text-destructive break-all">{file.name}</p>
+									<p class="text-xs text-muted-foreground mt-1">
+										{fileErrorData.modalityReasons[file.name] || 'Not supported by current model'}
+									</p>
+								</div>
+							{/each}
+						</div>
+					</div>
+				{/if}
+				
+				<!-- Supported file types -->
+				<div class="rounded-md bg-muted/50 p-3">
+					<h4 class="text-sm font-medium mb-2">This model supports:</h4>
+					<p class="text-sm text-muted-foreground">
+						{fileErrorData.supportedTypes.join(', ')}
+					</p>
+				</div>
+			</div>
+			
+			<AlertDialog.Footer>
+				<AlertDialog.Action onclick={() => showFileErrorDialog = false}>
+					Got it
+				</AlertDialog.Action>
+			</AlertDialog.Footer>
+		</AlertDialog.Content>
+	</AlertDialog.Portal>
+</AlertDialog.Root>
+
 <style>
 	.conversation-chat-form {
 		position: relative;

tools/server/webui/src/lib/components/app/chat/ChatSettings/ChatSettingsDialog.svelte

@@ -12,6 +12,7 @@
 		updateMultipleConfig,
 		resetConfig
 	} from '$lib/stores/settings.svelte';
+	import { supportsVision } from '$lib/stores/server.svelte';
 
 	interface Props {
 		onOpenChange?: (open: boolean) => void;
@@ -205,10 +206,12 @@
 										</p>
 									{/if}
 								{:else if field.type === 'checkbox'}
+									{@const isDisabled = field.key === 'pdfAsImage' && !supportsVision()}
 									<div class="flex items-start space-x-3">
 										<Checkbox
 											id={field.key}
 											checked={Boolean(localConfig[field.key])}
+											disabled={isDisabled}
 											onCheckedChange={(checked) =>
 												(localConfig[field.key] = checked)}
 											class="mt-1"
@@ -217,7 +220,7 @@
 										<div class="space-y-1">
 											<label
 												for={field.key}
-												class="cursor-pointer text-sm font-medium leading-none"
+												class="cursor-pointer text-sm font-medium leading-none {isDisabled ? 'text-muted-foreground' : ''}"
 											>
 												{field.label}
 											</label>
@@ -226,6 +229,10 @@
 												<p class="text-muted-foreground text-xs">
 													{field.help}
 												</p>
+											{:else if field.key === 'pdfAsImage' && !supportsVision()}
+												<p class="text-muted-foreground text-xs">
+													PDF-to-image processing requires a vision-capable model. PDFs will be processed as text.
+												</p>
 											{/if}
 										</div>
 									</div>

tools/server/webui/src/lib/utils/convert-files-to-extra.ts

@@ -1,13 +1,11 @@
 import { convertPDFToImage, convertPDFToText, isPdfMimeType } from "./pdf-processing";
 import { isSvgMimeType, svgBase64UrlToPngDataURL } from "./svg-to-png";
 import { isWebpMimeType, webpBase64UrlToPngDataURL } from "./webp-to-png";
-import { config } from '$lib/stores/settings.svelte';
-import { isLikelyTextFile, readFileAsText } from "./text-files";
-import { 
-    FileTypeCategory, 
-    AudioMimeType, 
-    getFileTypeCategory 
-} from '$lib/constants/supported-file-types';
+import { config, settingsStore } from '$lib/stores/settings.svelte';
+import { supportsVision } from '$lib/stores/server.svelte';
+import { FileTypeCategory, getFileTypeCategory } from '$lib/constants/supported-file-types';
+import { readFileAsText, isLikelyTextFile } from './text-files';
+import { toast } from 'svelte-sonner';
 
 function readFileAsBase64(file: File): Promise<string> {
     return new Promise((resolve, reject) => {
@@ -87,10 +85,28 @@ export async function parseFilesToMessageExtras(
                 // Always get base64 data for preview functionality
                 const base64Data = await readFileAsBase64(file.file);
                 const currentConfig = config();
-                const shouldProcessAsImages = Boolean(currentConfig.pdfAsImage);
+                const hasVisionSupport = supportsVision();
+                
+                // Force PDF-to-text for non-vision models
+                let shouldProcessAsImages = Boolean(currentConfig.pdfAsImage) && hasVisionSupport;
+                
+                // If user had pdfAsImage enabled but model doesn't support vision, update setting and notify
+                if (currentConfig.pdfAsImage && !hasVisionSupport) {
+                    console.log('Non-vision model detected: forcing PDF-to-text mode and updating settings');
+                    
+                    // Update the setting in localStorage
+                    settingsStore.updateConfig('pdfAsImage', false);
+                    
+                    // Show toast notification to user
+                    toast.warning('PDF setting changed: Non-vision model detected, PDFs will be processed as text instead of images.', {
+                        duration: 5000
+                    });
+                    
+                    shouldProcessAsImages = false;
+                }
 
                 if (shouldProcessAsImages) {
-                    // Process PDF as images
+                    // Process PDF as images (only for vision models)
                     try {
                         const images = await convertPDFToImage(file.file);
 
@@ -117,7 +133,7 @@ export async function parseFilesToMessageExtras(
                         });
                     }
                 } else {
-                    // Process PDF as text (default)
+                    // Process PDF as text (default or forced for non-vision models)
                     const content = await convertPDFToText(file.file);
 
                     extras.push({