allow image urls in image-reader

Author: Jicheng Lu

src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs

@@ -23,8 +23,8 @@ public async Task<bool> Execute(RoleDialogModel message)
         var agentService = _services.GetRequiredService<IAgentService>();
 
         var wholeDialogs = conv.GetDialogHistory();
-        var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
-        var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
+        var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs);
+        var agent = await agentService.LoadAgent(message.CurrentAgentId ?? BuiltInAgentId.UtilityAssistant);
         var fileAgent = new Agent
         {
             Id = agent?.Id ?? Guid.Empty.ToString(),
@@ -38,7 +38,7 @@ public async Task<bool> Execute(RoleDialogModel message)
         return true;
     }
 
-    private List<RoleDialogModel> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
+    private List<RoleDialogModel> AssembleFiles(string conversationId, IEnumerable<string>? imageUrls, List<RoleDialogModel> dialogs)
     {
         if (dialogs.IsNullOrEmpty())
         {
@@ -66,6 +66,18 @@ private List<RoleDialogModel> AssembleFiles(string conversationId, List<RoleDial
             }).ToList();
         }
 
+        if (!imageUrls.IsNullOrEmpty())
+        {
+            var lastDialog = dialogs.Last();
+            var files = lastDialog.Files ?? [];
+            var addnFiles = imageUrls.Select(x => x?.Trim())
+                                     .Where(x => !string.IsNullOrWhiteSpace(x))
+                                     .Select(x => new BotSharpFile { FileUrl = x }).ToList();
+            
+            files.AddRange(addnFiles);
+            lastDialog.Files = files;
+        }
+
         return dialogs;
     }
 

src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs

@@ -11,4 +11,12 @@ public class LlmContextIn
     [JsonPropertyName("image_description")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? ImageDescription { get; set; }
+
+    //[JsonPropertyName("image_url")]
+    //[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    //public string? ImageUrl { get; set; }
+
+    [JsonPropertyName("image_urls")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public IEnumerable<string>? ImageUrls { get; set; }
 }

src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json

@@ -7,6 +7,14 @@
       "user_request": {
         "type": "string",
         "description": "The request posted by user, which is related to analyzing requested images. User can request for multiple images to process at one time."
+      },
+      "image_urls": {
+        "type": "array",
+        "description": "The image, photo or picture urls that user requests for analysis. They typically start with 'http' or 'https'. If user doesn't include any url, then leave this array empty. Please remove any duplicated urls",
+        "items": {
+          "type": "string",
+          "description": "The image, photo or picture url that user requests for analysis. It typically starts with http or https."
+        }
       }
     },
     "required": [ "user_request" ]