Langchain::Assistant when using MistralAI accepts a message with image_url (#803)

* Langchain::Assistant when using MistralAI accepts a message with image_url

* fix linting errors

Author: andreibondarev

CHANGELOG.md

@@ -1,5 +1,5 @@
 ## [Unreleased]
-- Assistant can now process image_urls in the messages (currently only for OpenAI)
+- Assistant can now process image_urls in the messages (currently only for OpenAI and Mistral AI)
 
 ## [0.16.1] - 2024-09-30
 - Deprecate Langchain::LLM::GooglePalm

lib/langchain/assistants/assistant.rb

@@ -569,9 +569,7 @@ def build_chat_params(tools:, instructions:, messages:, tool_choice:)
           end
 
           def build_message(role:, content: nil, image_url: nil, tool_calls: [], tool_call_id: nil)
-            warn "Image URL is not supported by MistralAI currently" if image_url
-
-            Langchain::Messages::MistralAIMessage.new(role: role, content: content, tool_calls: tool_calls, tool_call_id: tool_call_id)
+            Langchain::Messages::MistralAIMessage.new(role: role, content: content, image_url: image_url, tool_calls: tool_calls, tool_call_id: tool_call_id)
           end
 
           # Extract the tool call information from the OpenAI tool call hash

lib/langchain/assistants/messages/mistral_ai_message.rb

@@ -15,17 +15,20 @@ class MistralAIMessage < Base
 
       # Initialize a new MistralAI message
       #
-      # @param [String] The role of the message
-      # @param [String] The content of the message
-      # @param [Array<Hash>] The tool calls made in the message
-      # @param [String] The ID of the tool call
-      def initialize(role:, content: nil, tool_calls: [], tool_call_id: nil) # TODO: Implement image_file: reference (https://platform.openai.com/docs/api-reference/messages/object#messages/object-content)
+      # @param role [String] The role of the message
+      # @param content [String] The content of the message
+      # @param image_url [String] The URL of the image
+      # @param tool_calls [Array<Hash>] The tool calls made in the message
+      # @param tool_call_id [String] The ID of the tool call
+      def initialize(role:, content: nil, image_url: nil, tool_calls: [], tool_call_id: nil) # TODO: Implement image_file: reference (https://platform.openai.com/docs/api-reference/messages/object#messages/object-content)
         raise ArgumentError, "Role must be one of #{ROLES.join(", ")}" unless ROLES.include?(role)
         raise ArgumentError, "Tool calls must be an array of hashes" unless tool_calls.is_a?(Array) && tool_calls.all? { |tool_call| tool_call.is_a?(Hash) }
 
         @role = role
         # Some Tools return content as a JSON hence `.to_s`
         @content = content.to_s
+        # Make sure you're using the Pixtral model if you want to send image_url
+        @image_url = image_url
         @tool_calls = tool_calls
         @tool_call_id = tool_call_id
       end
@@ -43,9 +46,28 @@ def llm?
       def to_hash
         {}.tap do |h|
           h[:role] = role
-          h[:content] = content if content # Content is nil for tool calls
-          h[:tool_calls] = tool_calls if tool_calls.any?
-          h[:tool_call_id] = tool_call_id if tool_call_id
+
+          if tool_calls.any?
+            h[:tool_calls] = tool_calls
+          else
+            h[:tool_call_id] = tool_call_id if tool_call_id
+
+            h[:content] = []
+
+            if content && !content.empty?
+              h[:content] << {
+                type: "text",
+                text: content
+              }
+            end
+
+            if image_url
+              h[:content] << {
+                type: "image_url",
+                image_url: image_url
+              }
+            end
+          end
         end
       end
 

spec/langchain/assistants/assistant_spec.rb

@@ -523,6 +523,19 @@
 
         thread.add_message(role: "user", content: "foo")
       end
+
+      it "adds a message with image_url" do
+        message_with_image = {role: "user", content: "hello", image_url: "https://example.com/image.jpg"}
+        subject = described_class.new(llm: llm, messages: [])
+
+        expect {
+          subject.add_message(**message_with_image)
+        }.to change { subject.messages.count }.from(0).to(1)
+        expect(subject.messages.first).to be_a(Langchain::Messages::MistralAIMessage)
+        expect(subject.messages.first.role).to eq("user")
+        expect(subject.messages.first.content).to eq("hello")
+        expect(subject.messages.first.image_url).to eq("https://example.com/image.jpg")
+      end
     end
 
     describe "#submit_tool_output" do
@@ -568,8 +581,8 @@
           allow(subject.llm).to receive(:chat)
             .with(
               messages: [
-                {role: "system", content: instructions},
-                {role: "user", content: "Please calculate 2+2"}
+                {role: "system", content: [{type: "text", text: instructions}]},
+                {role: "user", content: [{type: "text", text: "Please calculate 2+2"}]}
               ],
               tools: calculator.class.function_schemas.to_openai_format,
               tool_choice: "auto"
@@ -612,16 +625,16 @@
           allow(subject.llm).to receive(:chat)
             .with(
               messages: [
-                {role: "system", content: instructions},
-                {role: "user", content: "Please calculate 2+2"},
-                {role: "assistant", content: "", tool_calls: [
+                {role: "system", content: [{type: "text", text: instructions}]},
+                {role: "user", content: [{type: "text", text: "Please calculate 2+2"}]},
+                {role: "assistant", tool_calls: [
                   {
                     "function" => {"arguments" => "{\"input\":\"2+2\"}", "name" => "langchain_tool_calculator__execute"},
                     "id" => "call_9TewGANaaIjzY31UCpAAGLeV",
                     "type" => "function"
                   }
                 ]},
-                {content: "4.0", role: "tool", tool_call_id: "call_9TewGANaaIjzY31UCpAAGLeV"}
+                {content: [{type: "text", text: "4.0"}], role: "tool", tool_call_id: "call_9TewGANaaIjzY31UCpAAGLeV"}
               ],
               tools: calculator.class.function_schemas.to_openai_format,
               tool_choice: "auto"

spec/langchain/assistants/messages/mistral_ai_message_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+RSpec.describe Langchain::Messages::MistralAIMessage do
+  it "raises an error if role is not one of allowed" do
+    expect { described_class.new(role: "foo") }.to raise_error(ArgumentError)
+  end
+
+  describe "#to_hash" do
+    context "when role and content are not nil" do
+      let(:message) { described_class.new(role: "user", content: "Hello, world!", tool_calls: [], tool_call_id: nil) }
+
+      it "returns a hash with the role and content key" do
+        expect(message.to_hash).to eq({role: "user", content: [{type: "text", text: "Hello, world!"}]})
+      end
+    end
+
+    context "when tool_call_id is not nil" do
+      let(:message) { described_class.new(role: "tool", content: "Hello, world!", tool_calls: [], tool_call_id: "123") }
+
+      it "returns a hash with the tool_call_id key" do
+        expect(message.to_hash).to eq({role: "tool", content: [{type: "text", text: "Hello, world!"}], tool_call_id: "123"})
+      end
+    end
+
+    context "when tool_calls is not empty" do
+      let(:tool_call) {
+        {"id" => "call_9TewGANaaIjzY31UCpAAGLeV",
+         "type" => "function",
+         "function" => {"name" => "weather__execute", "arguments" => "{\"input\":\"Saint Petersburg\"}"}}
+      }
+
+      let(:message) { described_class.new(role: "assistant", tool_calls: [tool_call], tool_call_id: nil) }
+
+      it "returns a hash with the tool_calls key" do
+        expect(message.to_hash).to eq({role: "assistant", tool_calls: [tool_call]})
+      end
+    end
+
+    context "when image_url is present" do
+      let(:message) { described_class.new(role: "user", content: "Please describe this image", image_url: "https://example.com/image.jpg") }
+
+      it "returns a hash with the image_url key" do
+        expect(message.to_hash).to eq({
+          role: "user",
+          content: [
+            {type: "text", text: "Please describe this image"},
+            {type: "image_url", image_url: "https://example.com/image.jpg"}
+          ]
+        })
+      end
+    end
+  end
+end