open ai and claude are now working

Author: SamSaffron

lib/completions/dialects/chat_gpt.rb

@@ -102,35 +102,46 @@ def tool_msg(msg)
         end
 
         def user_msg(msg)
-          user_message = { role: "user", content: msg[:content] }
+          content_array = []
+
+          user_message = { role: "user" }
 
           if msg[:id]
             if @embed_user_ids
-              user_message[:content] = "#{msg[:id]}: #{msg[:content]}"
+              content_array << "#{msg[:id]}: "
             else
               user_message[:name] = msg[:id]
             end
           end
 
-          user_message[:content] = inline_images(user_message[:content], msg) if vision_support?
+          content_array << msg[:content]
+
+          content_array =
+            to_encoded_content_array(
+              content: content_array.flatten,
+              image_encoder: ->(details) { image_node(details) },
+              text_encoder: ->(text) { { type: "text", text: text } },
+              allow_vision: vision_support?,
+            )
+
+          user_message[:content] = no_array_if_only_text(content_array)
           user_message
         end
 
-        def inline_images(content, message)
-          encoded_uploads = prompt.encoded_uploads(message)
-          return content if encoded_uploads.blank?
-
-          content_w_imgs =
-            encoded_uploads.reduce([]) do |memo, details|
-              memo << {
-                type: "image_url",
-                image_url: {
-                  url: "data:#{details[:mime_type]};base64,#{details[:base64]}",
-                },
-              }
-            end
+        def no_array_if_only_text(content_array)
+          if content_array.size == 1 && content_array.first[:type] == "text"
+            return content_array.first[:text]
+          end
+          content_array
+        end
 
-          content_w_imgs << { type: "text", text: message[:content] }
+        def image_node(details)
+          {
+            type: "image_url",
+            image_url: {
+              url: "data:#{details[:mime_type]};base64,#{details[:base64]}",
+            },
+          }
         end
 
         def per_message_overhead

lib/completions/dialects/claude.rb

@@ -87,9 +87,9 @@ def tool_msg(msg)
         end
 
         def model_msg(msg)
-          if msg[:thinking] || msg[:redacted_thinking_signature]
-            content_array = []
+          content_array = []
 
+          if msg[:thinking] || msg[:redacted_thinking_signature]
             if msg[:thinking]
               content_array << {
                 type: "thinking",
@@ -104,13 +104,17 @@ def model_msg(msg)
                 data: msg[:redacted_thinking_signature],
               }
             end
+          end
 
-            content_array << { type: "text", text: msg[:content] }
+          content_array =
+            to_encoded_content_array(
+              content: [content_array, msg[:content]].flatten,
+              image_encoder: ->(details) { image_node(details) },
+              text_encoder: ->(text) { { type: "text", text: text } },
+              allow_vision: vision_support?,
+            )
 
-            { role: "assistant", content: content_array }
-          else
-            { role: "assistant", content: msg[:content] }
-          end
+          { role: "assistant", content: no_array_if_only_text(content_array) }
         end
 
         def system_msg(msg)
@@ -124,32 +128,31 @@ def system_msg(msg)
         end
 
         def user_msg(msg)
-          content = +""
-          content << "#{msg[:id]}: " if msg[:id]
-          message_content = msg[:content]
-          message_content = [message_content] if !message_content.is_a?(Array)
-
           content_array = []
+          content_array << "#{msg[:id]}: " if msg[:id]
+          content_array.concat([msg[:content]].flatten)
+
+          content_array =
+            to_encoded_content_array(
+              content: content_array,
+              image_encoder: ->(details) { image_node(details) },
+              text_encoder: ->(text) { { type: "text", text: text } },
+              allow_vision: vision_support?,
+            )
+
+          { role: "user", content: no_array_if_only_text(content_array) }
+        end
 
-          message_content.each do |content_part|
-            if content_part.is_a?(String)
-              content << content_part
-            elsif content_part.is_a?(Hash) && vision_support?
-              content_array << { type: "text", text: content } if content.present?
-              image = image_node(content_part[:upload_id])
-              content_array << image if image
-              content = +""
-            end
+        # keeping our payload as backward compatible as possible
+        def no_array_if_only_text(content_array)
+          if content_array.length == 1 && content_array.first[:type] == "text"
+            content_array.first[:text]
+          else
+            content_array
           end
-
-          content_array << { type: "text", text: content } if content.present?
-
-          { role: "user", content: content_array }
         end
 
-        def image_node(upload_id)
-          details = prompt.encode_upload(upload_id)
-          return nil if details.blank?
+        def image_node(details)
           {
             source: {
               type: "base64",

lib/completions/dialects/dialect.rb

@@ -227,6 +227,29 @@ def tool_msg(msg)
           msg = msg.merge(content: new_content)
           user_msg(msg)
         end
+
+        def to_encoded_content_array(content:, image_encoder:, text_encoder:, allow_vision:)
+          content = [content] if !content.is_a?(Array)
+
+          current_string = +""
+          result = []
+
+          content.each do |c|
+            if c.is_a?(String)
+              current_string << c
+            elsif c.is_a?(Hash) && c.key?(:upload_id) && allow_vision
+              if !current_string.empty?
+                result << text_encoder.call(current_string)
+                current_string = +""
+              end
+              encoded = prompt.encode_upload(c[:upload_id])
+              result << image_encoder.call(encoded) if encoded
+            end
+          end
+
+          result << text_encoder.call(current_string) if !current_string.empty?
+          result
+        end
       end
     end
   end

lib/completions/prompt_messages_builder.rb

@@ -216,12 +216,19 @@ def to_a(limit: nil, style: nil)
 
             if message[:type] == :user
               old_name = last_message.delete(:name)
-              last_message[:content] = "#{old_name}: #{last_message[:content]}" if old_name
+              last_message[:content] = ["#{old_name}: ", last_message[:content]].flatten if old_name
 
               new_content = message[:content]
-              new_content = "#{message[:name]}: #{new_content}" if message[:name]
+              new_content = ["#{message[:name]}: ", new_content].flatten if message[:name]
 
-              last_message[:content] += "\n#{new_content}"
+              if !last_message[:content].is_a?(Array)
+                last_message[:content] = [last_message[:content]]
+              end
+              last_message[:content].concat(["\n", new_content].flatten)
+
+              compressed =
+                compress_messages_buffer(last_message[:content], max_uploads: MAX_TOPIC_UPLOADS)
+              last_message[:content] = compressed
             else
               last_message[:content] = message[:content]
             end

spec/lib/completions/prompt_messages_builder_spec.rb

@@ -14,6 +14,25 @@
     Fabricate(:upload, user: user, original_filename: "image.png", extension: "png")
   end
 
+  it "correctly merges user messages with uploads" do
+    builder.push(type: :user, content: "Hello", name: "Alice", upload_ids: [1])
+    builder.push(type: :user, content: "World", name: "Bob", upload_ids: [2])
+
+    messages = builder.to_a
+
+    # Check the structure of the merged message
+    expect(messages.length).to eq(1)
+    expect(messages[0][:type]).to eq(:user)
+
+    # The content should contain the text and both uploads
+    content = messages[0][:content]
+    expect(content).to be_an(Array)
+    expect(content[0]).to eq("Alice: Hello")
+    expect(content[1]).to eq({ upload_id: 1 })
+    expect(content[2]).to eq("\nBob: World")
+    expect(content[3]).to eq({ upload_id: 2 })
+  end
+
   it "should allow merging user messages" do
     builder.push(type: :user, content: "Hello", name: "Alice")
     builder.push(type: :user, content: "World", name: "Bob")