image support is improved but not fully fixed yet

partially working in anthropic, still got quite a few dialects to go

Author: SamSaffron

lib/completions/dialects/claude.rb

@@ -126,29 +126,38 @@ def system_msg(msg)
         def user_msg(msg)
           content = +""
           content << "#{msg[:id]}: " if msg[:id]
-          content << msg[:content]
-          content = inline_images(content, msg) if vision_support?
+          message_content = msg[:content]
+          message_content = [message_content] if !message_content.is_a?(Array)
+
+          content_array = []
+
+          message_content.each do |content_part|
+            if content_part.is_a?(String)
+              content << content_part
+            elsif content_part.is_a?(Hash) && vision_support?
+              content_array << { type: "text", text: content } if content.present?
+              image = image_node(content_part[:upload_id])
+              content_array << image if image
+              content = +""
+            end
+          end
 
-          { role: "user", content: content }
-        end
+          content_array << { type: "text", text: content } if content.present?
 
-        def inline_images(content, message)
-          encoded_uploads = prompt.encoded_uploads(message)
-          return content if encoded_uploads.blank?
-
-          content_w_imgs =
-            encoded_uploads.reduce([]) do |memo, details|
-              memo << {
-                source: {
-                  type: "base64",
-                  data: details[:base64],
-                  media_type: details[:mime_type],
-                },
-                type: "image",
-              }
-            end
+          { role: "user", content: content_array }
+        end
 
-          content_w_imgs << { type: "text", text: content }
+        def image_node(upload_id)
+          details = prompt.encode_upload(upload_id)
+          return nil if details.blank?
+          {
+            source: {
+              type: "base64",
+              data: details[:base64],
+              media_type: details[:mime_type],
+            },
+            type: "image",
+          }
         end
       end
     end

lib/completions/prompt_messages_builder.rb

@@ -280,51 +280,49 @@ def push(type:, content:, name: nil, upload_ids: nil, id: nil, thinking: nil)
 
       def topic_array
         raw_messages = @raw_messages.dup
-        user_content = +"You are operating in a Discourse forum.\n\n"
+        content_array = []
+        content_array << "You are operating in a Discourse forum.\n\n"
 
         if @topic
           if @topic.private_message?
-            user_content << "Private message info.\n"
+            content_array << "Private message info.\n"
           else
-            user_content << "Topic information:\n"
+            content_array << "Topic information:\n"
           end
 
-          user_content << "- URL: #{@topic.url}\n"
-          user_content << "- Title: #{@topic.title}\n"
+          content_array << "- URL: #{@topic.url}\n"
+          content_array << "- Title: #{@topic.title}\n"
           if SiteSetting.tagging_enabled
             tags = @topic.tags.pluck(:name)
             tags -= DiscourseTagging.hidden_tag_names if tags.present?
-            user_content << "- Tags: #{tags.join(", ")}\n" if tags.present?
+            content_array << "- Tags: #{tags.join(", ")}\n" if tags.present?
           end
           if !@topic.private_message?
-            user_content << "- Category: #{@topic.category.name}\n" if @topic.category
+            content_array << "- Category: #{@topic.category.name}\n" if @topic.category
           end
-          user_content << "- Number of replies: #{@topic.posts_count - 1}\n\n"
+          content_array << "- Number of replies: #{@topic.posts_count - 1}\n\n"
         end
 
         last_user_message = raw_messages.pop
 
-        upload_ids = []
         if raw_messages.present?
-          user_content << "Here is the conversation so far:\n"
+          content_array << "Here is the conversation so far:\n"
           raw_messages.each do |message|
-            user_content << "#{message[:name] || "User"}: #{message[:content]}\n"
-            upload_ids.concat(message[:upload_ids]) if message[:upload_ids].present?
+            content_array << "#{message[:name] || "User"}: "
+            content_array << message[:content]
+            content_array << "\n\n"
           end
         end
 
         if last_user_message
-          user_content << "You are responding to #{last_user_message[:name] || "User"} who just said:\n #{last_user_message[:content]}"
-          if last_user_message[:upload_ids].present?
-            upload_ids.concat(last_user_message[:upload_ids])
-          end
+          content_array << "You are responding to #{last_user_message[:name] || "User"} who just said:\n"
+          content_array << last_user_message[:content]
         end
 
-        user_message = { type: :user, content: user_content }
+        content_array =
+          compress_messages_buffer(content_array.flatten, max_uploads: MAX_TOPIC_UPLOADS)
 
-        if upload_ids.present?
-          user_message[:upload_ids] = upload_ids[-MAX_TOPIC_UPLOADS..-1] || upload_ids
-        end
+        user_message = { type: :user, content: content_array }
 
         [user_message]
       end
@@ -388,6 +386,8 @@ def compress_messages_buffer(buffer, max_uploads:)
           compressed.delete_if { |item| item.is_a?(Hash) && (counter += 1) > 0 }
         end
 
+        compressed = compressed[0] if compressed.length == 1 && compressed[0].is_a?(String)
+
         compressed
       end
     end

spec/lib/completions/prompt_messages_builder_spec.rb

@@ -3,9 +3,17 @@
 describe DiscourseAi::Completions::PromptMessagesBuilder do
   let(:builder) { DiscourseAi::Completions::PromptMessagesBuilder.new }
   fab!(:user)
+  fab!(:admin)
   fab!(:bot_user) { Fabricate(:user) }
   fab!(:other_user) { Fabricate(:user) }
 
+  fab!(:image_upload1) do
+    Fabricate(:upload, user: user, original_filename: "image.png", extension: "png")
+  end
+  fab!(:image_upload2) do
+    Fabricate(:upload, user: user, original_filename: "image.png", extension: "png")
+  end
+
   it "should allow merging user messages" do
     builder.push(type: :user, content: "Hello", name: "Alice")
     builder.push(type: :user, content: "World", name: "Bob")
@@ -239,6 +247,60 @@
       )
     end
 
+    it "handles uploads correctly in topic style messages" do
+      # Use Discourse's upload format in the post raw content
+      upload_markdown = "![test|658x372](#{image_upload1.short_url})"
+
+      post_with_upload =
+        Fabricate(
+          :post,
+          topic: pm,
+          user: admin,
+          raw: "This is the original #{upload_markdown} I just added",
+        )
+
+      UploadReference.create!(target: post_with_upload, upload: image_upload1)
+
+      upload2_markdown = "![test|658x372](#{image_upload2.short_url})"
+
+      post2_with_upload =
+        Fabricate(
+          :post,
+          topic: pm,
+          user: admin,
+          raw: "This post has a different image #{upload2_markdown} I just added",
+        )
+
+      UploadReference.create!(target: post2_with_upload, upload: image_upload2)
+
+      messages =
+        described_class.messages_from_post(
+          post2_with_upload,
+          style: :topic,
+          max_posts: 3,
+          bot_usernames: [bot_user.username],
+          include_uploads: true,
+        )
+
+      # this is not quite ideal yet, images are attached at the end of the post
+      # long term we may want to extract them out using a regex and create N parts
+      # so people can talk about multiple images in a single post
+      # this is the initial ground work though
+
+      expect(messages.length).to eq(1)
+      content = messages[0][:content]
+
+      # first part
+      # first image
+      # second part
+      # second image
+      expect(content.length).to eq(4)
+      expect(content[0]).to include("This is the original")
+      expect(content[1]).to eq({ upload_id: image_upload1.id })
+      expect(content[2]).to include("different image")
+      expect(content[3]).to eq({ upload_id: image_upload2.id })
+    end
+
     context "with limited context" do
       it "respects max_context_posts" do
         context =