Added support for images (#609)

Author: SommerEngineering

app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs

@@ -159,7 +159,9 @@ public async Task<List<SelectedDataSource>> PerformSelectionAsync(IProvider prov
             ContentText text => text.Text,
 
             // Image prompts may be empty, e.g., when the image is too large:
-            ContentImage image => await image.AsBase64(token),
+            ContentImage image => await image.TryAsBase64(token) is (success: true, { } base64Image)
+                ? base64Image 
+                : string.Empty,
 
             // Other content types are not supported yet:
             _ => string.Empty,

app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs

@@ -219,7 +219,9 @@ public async Task<RetrievalContextValidationResult> ValidateRetrievalContextAsyn
                 ContentText text => text.Text,
 
                 // Image prompts may be empty, e.g., when the image is too large:
-                ContentImage image => await image.AsBase64(token),
+                ContentImage image => await image.TryAsBase64(token) is (success: true, { } base64Image)
+                    ? base64Image 
+                    : string.Empty,
 
                 // Other content types are not supported yet:
                 _ => string.Empty,

app/MindWork AI Studio/Assistants/AssistantBase.razor.cs

@@ -217,12 +217,13 @@ protected Guid CreateChatThread(Guid workspaceId, string name)
         return chatId;
     }
 
-    protected DateTimeOffset AddUserRequest(string request, bool hideContentFromUser = false)
+    protected DateTimeOffset AddUserRequest(string request, bool hideContentFromUser = false, params List<FileAttachment> attachments)
     {
         var time = DateTimeOffset.Now;
         this.lastUserPrompt = new ContentText
         {
             Text = request,
+            FileAttachments = attachments,
         };
 
         this.chatThread!.Blocks.Add(new ContentBlock

app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor

@@ -103,7 +103,7 @@ else
             @T("Documents for the analysis")
         </MudText>
 
-        <AttachDocuments Name="Document Analysis Files" @bind-DocumentPaths="@this.loadedDocumentPaths" CatchAllDocuments="true" UseSmallForm="false"/>
+        <AttachDocuments Name="Document Analysis Files" @bind-DocumentPaths="@this.loadedDocumentPaths" CatchAllDocuments="true" UseSmallForm="false" Provider="@this.providerSettings"/>
 
     </ExpansionPanel>
 </MudExpansionPanels>

app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor.cs

@@ -1,3 +1,5 @@
+using System.Text;
+
 using AIStudio.Chat;
 using AIStudio.Dialogs;
 using AIStudio.Dialogs.Settings;
@@ -34,19 +36,21 @@ You are a policy‑bound analysis agent. Follow these instructions exactly.
         
         DOCUMENTS: the only content you may analyze.
         
+        Maybe, there are image files attached. IMAGES may contain important information. Use them as part of your analysis.
+        
         {this.GetDocumentTaskDescription()}
         
         # Scope and precedence
         
-        Use only information explicitly contained in DOCUMENTS and/or POLICY_*.
+        Use only information explicitly contained in DOCUMENTS, IMAGES, and/or POLICY_*.
         You may paraphrase but must not add facts, assumptions, or outside knowledge.
         Content decisions are governed by POLICY_ANALYSIS_RULES; formatting is governed by POLICY_OUTPUT_RULES.
         If there is a conflict between DOCUMENTS and POLICY_*, follow POLICY_ANALYSIS_RULES for analysis and POLICY_OUTPUT_RULES for formatting. Do not invent reconciliations.
         
         # Process
         
         1) Read POLICY_ANALYSIS_RULES and POLICY_OUTPUT_RULES end to end.
-        2) Extract only the information from DOCUMENTS that POLICY_ANALYSIS_RULES permits.
+        2) Extract only the information from DOCUMENTS and IMAGES that POLICY_ANALYSIS_RULES permits.
         3) Perform the analysis strictly according to POLICY_ANALYSIS_RULES.
         4) Produce the final answer strictly according to POLICY_OUTPUT_RULES.
         
@@ -74,16 +78,33 @@ Do not quote or summarize POLICY_* unless required by POLICY_OUTPUT_RULES.
         # Self‑check before sending
         
         Verify the answer matches POLICY_OUTPUT_RULES exactly.
-        Verify every statement is attributable to DOCUMENTS or POLICY_*.
+        Verify every statement is attributable to DOCUMENTS, IMAGES, or POLICY_*.
         Remove any text not required by POLICY_OUTPUT_RULES.
         
         {this.PromptGetActivePolicy()}
         """;
 
-    private string GetDocumentTaskDescription() =>
-        this.loadedDocumentPaths.Count > 1
-            ? $"Your task is to analyze {this.loadedDocumentPaths.Count} DOCUMENTS. Different DOCUMENTS are divided by a horizontal rule in markdown formatting followed by the name of the document."
-            : "Your task is to analyze a single document.";
+    private string GetDocumentTaskDescription()
+    {
+        var numDocuments = this.loadedDocumentPaths.Count(x => x is { Exists: true, IsImage: false });
+        var numImages = this.loadedDocumentPaths.Count(x => x is { Exists: true, IsImage: true });
+        
+        return (numDocuments, numImages) switch
+        {
+            (0, 1) => "Your task is to analyze a single image file attached as a document.",
+            (0, > 1) => $"Your task is to analyze {numImages} image file(s) attached as documents.",
+            
+            (1, 0) => "Your task is to analyze a single DOCUMENT.",
+            (1, 1) => "Your task is to analyze a single DOCUMENT and 1 image file attached as a document.",
+            (1, > 1) => $"Your task is to analyze a single DOCUMENT and {numImages} image file(s) attached as documents.",
+            
+            (> 0, 0) => $"Your task is to analyze {numDocuments} DOCUMENTS. Different DOCUMENTS are divided by a horizontal rule in markdown formatting followed by the name of the document.",
+            (> 0, 1) => $"Your task is to analyze {numDocuments} DOCUMENTS and 1 image file attached as a document. Different DOCUMENTS are divided by a horizontal rule in Markdown formatting followed by the name of the document.",
+            (> 0, > 0) => $"Your task is to analyze {numDocuments} DOCUMENTS and {numImages} image file(s) attached as documents. Different DOCUMENTS are divided by a horizontal rule in Markdown formatting followed by the name of the document.",
+            
+            _ => "Your task is to analyze a single DOCUMENT."
+        };
+    }
 
     protected override IReadOnlyList<IButtonData> FooterButtons => [];
 
@@ -327,37 +348,68 @@ private async Task<string> PromptLoadDocumentsContent()
         if (this.loadedDocumentPaths.Count == 0)
             return string.Empty;
 
-        var documentSections = new List<string>();
-        var count = 1;
+        var documents = this.loadedDocumentPaths.Where(n => n is { Exists: true, IsImage: false }).ToList();
+        var sb = new StringBuilder();
+
+        if (documents.Count > 0)
+        {
+            sb.AppendLine("""
+                          # DOCUMENTS:
+
+                          """);
+        }
 
-        foreach (var fileAttachment in this.loadedDocumentPaths)
+        var numDocuments = 1;
+        foreach (var document in documents)
         {
-            if (fileAttachment.IsForbidden)
+            if (document.IsForbidden)
             {
-                this.Logger.LogWarning($"Skipping forbidden file: '{fileAttachment.FilePath}'.");
+                this.Logger.LogWarning($"Skipping forbidden file: '{document.FilePath}'.");
                 continue;
             }
-            
-            var fileContent = await this.RustService.ReadArbitraryFileData(fileAttachment.FilePath, int.MaxValue);
-
-            documentSections.Add($"""
-                                  ## DOCUMENT {count}:
-                                  File path: {fileAttachment.FilePath}
-                                  Content:
-                                  ```
-                                  {fileContent}
-                                  ```
-
-                                  ---
-                                  """);
-            count++;
+
+            var fileContent = await this.RustService.ReadArbitraryFileData(document.FilePath, int.MaxValue);
+            sb.AppendLine($"""
+                           
+                           ## DOCUMENT {numDocuments}:
+                           File path: {document.FilePath}
+                           Content:
+                           ```
+                           {fileContent}
+                           ```
+
+                           ---
+                           
+                           """);
+            numDocuments++;
         }
 
-        return $"""
-                # DOCUMENTS:
+        var numImages = this.loadedDocumentPaths.Count(x => x is { IsImage: true, Exists: true });
+        if (numImages > 0)
+        {
+            if (documents.Count == 0)
+            {
+                sb.AppendLine($"""
+
+                               There are {numImages} image file(s) attached as documents.
+                               Please consider them as documents as well and use them to
+                               answer accordingly.
+
+                               """);
+            }
+            else
+            {
+                sb.AppendLine($"""
+
+                               Additionally, there are {numImages} image file(s) attached.
+                               Please consider them as documents as well and use them to
+                               answer accordingly.
+
+                               """);
+            }
+        }
 
-                {string.Join("\n", documentSections)}
-                """;
+        return sb.ToString();
     }
 
     private async Task Analyze()
@@ -370,7 +422,9 @@ private async Task Analyze()
         this.CreateChatThread();
 
         var userRequest = this.AddUserRequest(
-            $"{await this.PromptLoadDocumentsContent()}", hideContentFromUser:true);
+            await this.PromptLoadDocumentsContent(),
+            hideContentFromUser: true,
+            this.loadedDocumentPaths.Where(n => n is { Exists: true, IsImage: true }).ToList());
 
         await this.AddAIResponseAsync(userRequest);
     }

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -1513,6 +1513,12 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T4188329028"] = "No, kee
 -- Export Chat to Microsoft Word
 UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTBLOCKCOMPONENT::T861873672"] = "Export Chat to Microsoft Word"
 
+-- The local image file does not exist. Skipping the image.
+UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T255679918"] = "The local image file does not exist. Skipping the image."
+
+-- Failed to download the image from the URL. Skipping the image.
+UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T2996654916"] = "Failed to download the image from the URL. Skipping the image."
+
 -- The local image file is too large (>10 MB). Skipping the image.
 UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T3219823625"] = "The local image file is too large (>10 MB). Skipping the image."
 
@@ -2968,6 +2974,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1373123357"] = "Markdo
 -- Load file
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Load file"
 
+-- Image View
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2199753423"] = "Image View"
+
 -- See how we load your file. Review the content before we process it further.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "See how we load your file. Review the content before we process it further."
 
@@ -2986,6 +2995,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T652739927"] = "This is
 -- File Path
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "File Path"
 
+-- The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T973777830"] = "The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible."
+
 -- Embedding Name
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Embedding Name"
 
@@ -3436,12 +3448,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T1746160064"] = "He
 -- There aren't any file attachments available right now.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T2111340711"] = "There aren't any file attachments available right now."
 
+-- Document Preview
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T285154968"] = "Document Preview"
+
 -- The file was deleted, renamed, or moved.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3083729256"] = "The file was deleted, renamed, or moved."
 
 -- Your attached file.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3154198222"] = "Your attached file."
 
+-- Preview what we send to the AI.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3160778981"] = "Preview what we send to the AI."
+
+-- Close
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3448155331"] = "Close"
+
 -- Your attached files
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Your attached files"
 
@@ -6058,9 +6079,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T29289275
 -- Images are not supported yet
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T298062956"] = "Images are not supported yet"
 
+-- Images are not supported at this place
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T305247150"] = "Images are not supported at this place"
+
 -- Executables are not allowed
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T4167762413"] = "Executables are not allowed"
 
+-- Images are not supported by the selected provider and model
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::FILEEXTENSIONVALIDATION::T999194030"] = "Images are not supported by the selected provider and model"
+
 -- The hostname is not a valid HTTP(S) URL.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::PROVIDERVALIDATION::T1013354736"] = "The hostname is not a valid HTTP(S) URL."
 

app/MindWork AI Studio/Chat/ChatThread.cs

@@ -238,7 +238,7 @@ public void Remove(IContent content, bool removeForRegenerate = false)
         {
             var (contentData, contentType) = block.Content switch
             {
-                ContentImage image => (await image.AsBase64(token), Tools.ERIClient.DataModel.ContentType.IMAGE),
+                ContentImage image => (await image.TryAsBase64(token) is (success: true, { } base64Image) ? base64Image : string.Empty, Tools.ERIClient.DataModel.ContentType.IMAGE),
                 ContentText text => (text.Text, Tools.ERIClient.DataModel.ContentType.TEXT),
 
                 _ => (string.Empty, Tools.ERIClient.DataModel.ContentType.UNKNOWN),

app/MindWork AI Studio/Chat/ContentImage.cs

@@ -47,6 +47,8 @@ public Task<ChatThread> CreateFromProviderAsync(IProvider provider, Model chatMo
         InitialRemoteWait = this.InitialRemoteWait,
         IsStreaming = this.IsStreaming,
         SourceType = this.SourceType,
+        Sources = [..this.Sources],
+        FileAttachments = [..this.FileAttachments],
     };
 
     #endregion

app/MindWork AI Studio/Chat/ContentText.cs

@@ -195,6 +195,14 @@ public async Task<string> PrepareTextContentForAI()
                         sb.AppendLine(await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue));
                         sb.AppendLine("````");
                     }
+                    
+                    var numImages = this.FileAttachments.Count(x => x is { IsImage: true, Exists: true });
+                    if (numImages > 0)
+                    {
+                        sb.AppendLine();
+                        sb.AppendLine($"Additionally, there are {numImages} image file(s) attached to this message. ");
+                        sb.AppendLine("Please consider them as part of the message content and use them to answer accordingly.");
+                    }
                 }
             }
         }

app/MindWork AI Studio/Chat/FileAttachment.cs

@@ -1,3 +1,5 @@
+using System.Text.Json.Serialization;
+
 using AIStudio.Tools.Rust;
 
 namespace AIStudio.Chat;
@@ -9,22 +11,47 @@ namespace AIStudio.Chat;
 /// <param name="FileName">The name of the file, including extension.</param>
 /// <param name="FilePath">The full path to the file, including the filename and extension.</param>
 /// <param name="FileSizeBytes">The size of the file in bytes.</param>
-public readonly record struct FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes)
+[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
+[JsonDerivedType(typeof(FileAttachment), typeDiscriminator: "file")]
+[JsonDerivedType(typeof(FileAttachmentImage), typeDiscriminator: "image")]
+public record FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes)
 {
     /// <summary>
-    /// Gets a value indicating whether the file still exists on the file system.
+    /// Gets a value indicating whether the file type is forbidden and should not be attached.
     /// </summary>
-    public bool Exists => File.Exists(this.FilePath);
+    /// <remarks>
+    /// The state is determined once during construction and does not change.
+    /// </remarks>
+    public bool IsForbidden { get; } = Type == FileAttachmentType.FORBIDDEN;
 
     /// <summary>
-    /// Gets a value indicating whether the file type is forbidden and should not be attached.
+    /// Gets a value indicating whether the file type is valid and allowed to be attached.
     /// </summary>
-    public bool IsForbidden => this.Type == FileAttachmentType.FORBIDDEN;
+    /// <remarks>
+    /// The state is determined once during construction and does not change.
+    /// </remarks>
+    public bool IsValid { get; } = Type != FileAttachmentType.FORBIDDEN;
 
     /// <summary>
-    /// Gets a value indicating whether the file type is valid and allowed to be attached.
+    /// Gets a value indicating whether the file type is an image.
     /// </summary>
-    public bool IsValid => this.Type != FileAttachmentType.FORBIDDEN;
+    /// <remarks>
+    /// The state is determined once during construction and does not change.
+    /// </remarks>
+    public bool IsImage { get; } = Type == FileAttachmentType.IMAGE;
+
+    /// <summary>
+    /// Gets the file path for loading the file from the web browser-side (Blazor).
+    /// </summary>
+    public string FilePathAsUrl { get; } = FileHandler.CreateFileUrl(FilePath);
+    
+    /// <summary>
+    /// Gets a value indicating whether the file still exists on the file system.
+    /// </summary>
+    /// <remarks>
+    /// This property checks the file system each time it is accessed.
+    /// </remarks>
+    public bool Exists => File.Exists(this.FilePath);
 
     /// <summary>
     /// Creates a FileAttachment from a file path by automatically determining the type,
@@ -38,7 +65,13 @@ public static FileAttachment FromPath(string filePath)
         var fileSize = File.Exists(filePath) ? new FileInfo(filePath).Length : 0;
         var type = DetermineFileType(filePath);
 
-        return new FileAttachment(type, fileName, filePath, fileSize);
+        return type switch
+        {
+            FileAttachmentType.DOCUMENT => new FileAttachment(type, fileName, filePath, fileSize),
+            FileAttachmentType.IMAGE => new FileAttachmentImage(fileName, filePath, fileSize),
+            
+            _ => new FileAttachment(type, fileName, filePath, fileSize),
+        };
     }
 
     /// <summary>