Supporting image_urls

Author: TonsOfFun

lib/active_agent/action_prompt/message.rb

@@ -3,7 +3,7 @@ module ActionPrompt
     class Message
       VALID_ROLES = %w[system assistant user tool].freeze
 
-      attr_accessor :action_id, :action_name, :generation_id, :content, :role, :action_requested, :requested_actions, :content_type, :charset
+      attr_accessor :action_id, :action_name, :raw_actions, :generation_id, :content, :role, :action_requested, :requested_actions, :content_type, :charset
 
       def initialize(attributes = {})
         @action_id = attributes[:action_id]
@@ -13,6 +13,7 @@ def initialize(attributes = {})
         @content = attributes[:content] || ""
         @content_type = attributes[:content_type] || "text/plain"
         @role = attributes[:role] || :user
+        @raw_actions = attributes[:raw_actions]
         @requested_actions = attributes[:requested_actions] || []
         @action_requested = @requested_actions.any?
         validate_role

lib/active_agent/generation_provider/open_ai_provider.rb

@@ -73,13 +73,15 @@ def provider_messages(messages)
             role: message.role,
             tool_call_id: message.action_id.presence,
             name: message.action_name.presence,
-            tool_calls: (message.requested_actions.map { |action| {name: action.name, arguments: action.params.to_json} } if message.action_requested),
+            tool_calls: message.raw_actions.present? ? message.raw_actions[:tool_calls] : (message.requested_actions.map { |action| {type: "function", name: action.name, arguments: action.params.to_json} } if message.action_requested),
+            generation_id: message.generation_id,
             content: message.content,
             type: message.content_type,
             charset: message.charset
           }.compact
 
-          if message.content_type == "image_url"
+          if message.content_type == "image_url" || message.content[0..4] == "data:"
+            provider_message[:type] = "image_url"
             provider_message[:image_url] = {url: message.content}
           end
           provider_message
@@ -103,6 +105,7 @@ def handle_message(message_json)
           content: message_json["content"],
           role: message_json["role"].intern,
           action_requested: message_json["finish_reason"] == "tool_calls",
+          raw_actions: message_json["tool_calls"] || [],
           requested_actions: handle_actions(message_json["tool_calls"])
         )
       end