Skip to content

Commit 9502ce0

Browse files
committed
Fix paint not working with OpenRouter provider (#513)
OpenRouter uses the chat completions endpoint for image generation instead of a dedicated /images/generations endpoint. This commit adds an OpenRouter::Images module that: - Routes image generation requests to chat/completions - Formats the payload with modalities: ["image", "text"] - Parses the response from choices[0].message.images[] - Handles both base64 data URLs and regular URLs
1 parent 7a8e917 commit 9502ce0

File tree

5 files changed

+322
-0
lines changed

5 files changed

+322
-0
lines changed

lib/ruby_llm/providers/openrouter.rb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ module Providers
55
# OpenRouter API integration.
66
class OpenRouter < OpenAI
77
include OpenRouter::Models
8+
include OpenRouter::Images
89

910
def api_base
1011
'https://openrouter.ai/api/v1'
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
# frozen_string_literal: true
2+
3+
module RubyLLM
4+
module Providers
5+
class OpenRouter
6+
# Image generation methods for the OpenRouter API integration.
7+
# OpenRouter uses the chat completions endpoint for image generation
8+
# instead of a dedicated images endpoint.
9+
module Images
10+
module_function
11+
12+
def images_url
13+
'chat/completions'
14+
end
15+
16+
def render_image_payload(prompt, model:, size:)
17+
RubyLLM.logger.debug "Ignoring size #{size}. OpenRouter image generation does not support size parameter."
18+
{
19+
model: model,
20+
messages: [
21+
{
22+
role: 'user',
23+
content: prompt
24+
}
25+
],
26+
modalities: %w[image text]
27+
}
28+
end
29+
30+
def parse_image_response(response, model:)
31+
data = response.body
32+
message = data.dig('choices', 0, 'message')
33+
34+
unless message&.key?('images') && message['images']&.any?
35+
raise Error.new(nil, 'Unexpected response format from OpenRouter image generation API')
36+
end
37+
38+
image_data = message['images'].first
39+
image_url = image_data.dig('image_url', 'url') || image_data['url']
40+
41+
raise Error.new(nil, 'No image URL found in OpenRouter response') unless image_url
42+
43+
build_image_from_url(image_url, model)
44+
end
45+
46+
def build_image_from_url(image_url, model)
47+
if image_url.start_with?('data:')
48+
# Parse data URL format: data:image/png;base64,<data>
49+
match = image_url.match(/^data:([^;]+);base64,(.+)$/)
50+
raise Error.new(nil, 'Invalid data URL format from OpenRouter') unless match
51+
52+
Image.new(
53+
data: match[2],
54+
mime_type: match[1],
55+
model_id: model
56+
)
57+
else
58+
# Regular URL
59+
Image.new(
60+
url: image_url,
61+
mime_type: 'image/png',
62+
model_id: model
63+
)
64+
end
65+
end
66+
end
67+
end
68+
end
69+
end

spec/fixtures/vcr_cassettes/image_basic_functionality_openrouter_google_gemini-2_5-flash-image-preview_can_paint_images.yml

Lines changed: 56 additions & 0 deletions
Large diffs are not rendered by default.

spec/ruby_llm/image_generation_spec.rb

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,5 +75,21 @@ def save_and_verify_image(image)
7575

7676
save_and_verify_image image
7777
end
78+
79+
it 'openrouter/google/gemini-2.5-flash-image-preview can paint images' do
80+
image = RubyLLM.paint(
81+
'a siamese cat',
82+
model: 'google/gemini-2.5-flash-image-preview',
83+
provider: :openrouter,
84+
assume_model_exists: true
85+
)
86+
87+
expect(image.base64?).to be(true)
88+
expect(image.data).to be_present
89+
expect(image.mime_type).to include('image')
90+
expect(image.model_id).to eq('google/gemini-2.5-flash-image-preview')
91+
92+
save_and_verify_image image
93+
end
7894
end
7995
end
Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
# frozen_string_literal: true
2+
3+
require 'spec_helper'
4+
5+
RSpec.describe RubyLLM::Providers::OpenRouter::Images do
6+
let(:images_module) { described_class }
7+
8+
describe '.images_url' do
9+
it 'returns the chat completions endpoint' do
10+
expect(images_module.images_url).to eq('chat/completions')
11+
end
12+
end
13+
14+
describe '.render_image_payload' do
15+
it 'renders a chat completion payload with image modality' do
16+
payload = images_module.render_image_payload('a cute cat', model: 'test-model', size: '1024x1024')
17+
18+
expect(payload[:model]).to eq('test-model')
19+
expect(payload[:messages]).to eq([{ role: 'user', content: 'a cute cat' }])
20+
expect(payload[:modalities]).to eq(%w[image text])
21+
end
22+
23+
it 'ignores size parameter and logs debug message' do
24+
allow(RubyLLM.logger).to receive(:debug)
25+
images_module.render_image_payload('a cute cat', model: 'test-model', size: '512x512')
26+
expect(RubyLLM.logger).to have_received(:debug).with(/Ignoring size/)
27+
end
28+
end
29+
30+
describe '.parse_image_response' do
31+
context 'with base64 data URL response' do
32+
let(:response_body) do
33+
{
34+
'choices' => [{
35+
'message' => {
36+
'role' => 'assistant',
37+
'content' => 'Here is an image of a cat',
38+
'images' => [{
39+
'type' => 'image_url',
40+
'image_url' => {
41+
'url' => 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE='
42+
}
43+
}]
44+
}
45+
}]
46+
}
47+
end
48+
let(:response) { instance_double(Faraday::Response, body: response_body) }
49+
50+
it 'parses base64 image data from response' do
51+
image = images_module.parse_image_response(response, model: 'test-model')
52+
53+
expect(image).to be_a(RubyLLM::Image)
54+
expect(image.base64?).to be(true)
55+
expect(image.data).to eq('iVBORw0KGgoAAAANSUhEUgAAAAE=')
56+
expect(image.mime_type).to eq('image/png')
57+
expect(image.model_id).to eq('test-model')
58+
end
59+
end
60+
61+
context 'with jpeg image data URL' do
62+
let(:response_body) do
63+
{
64+
'choices' => [{
65+
'message' => {
66+
'images' => [{
67+
'image_url' => { 'url' => 'data:image/jpeg;base64,/9j/4AAQSkZJRg==' }
68+
}]
69+
}
70+
}]
71+
}
72+
end
73+
let(:response) { instance_double(Faraday::Response, body: response_body) }
74+
75+
it 'parses jpeg image data' do
76+
image = images_module.parse_image_response(response, model: 'test-model')
77+
78+
expect(image.mime_type).to eq('image/jpeg')
79+
expect(image.data).to eq('/9j/4AAQSkZJRg==')
80+
end
81+
end
82+
83+
context 'with direct url field (alternative format)' do
84+
let(:response_body) do
85+
{
86+
'choices' => [{
87+
'message' => {
88+
'images' => [{ 'url' => 'data:image/png;base64,abc123' }]
89+
}
90+
}]
91+
}
92+
end
93+
let(:response) { instance_double(Faraday::Response, body: response_body) }
94+
95+
it 'parses image from direct url field' do
96+
image = images_module.parse_image_response(response, model: 'test-model')
97+
98+
expect(image.data).to eq('abc123')
99+
end
100+
end
101+
102+
context 'with regular URL response' do
103+
let(:response_body) do
104+
{
105+
'choices' => [{
106+
'message' => {
107+
'images' => [{
108+
'image_url' => { 'url' => 'https://example.com/image.png' }
109+
}]
110+
}
111+
}]
112+
}
113+
end
114+
let(:response) { instance_double(Faraday::Response, body: response_body) }
115+
116+
it 'creates image with URL instead of base64 data' do
117+
image = images_module.parse_image_response(response, model: 'test-model')
118+
119+
expect(image.base64?).to be(false)
120+
expect(image.url).to eq('https://example.com/image.png')
121+
expect(image.mime_type).to eq('image/png')
122+
end
123+
end
124+
125+
context 'with missing images in response' do
126+
let(:response_body) do
127+
{
128+
'choices' => [{
129+
'message' => { 'content' => 'Sorry, I cannot generate images' }
130+
}]
131+
}
132+
end
133+
let(:response) { instance_double(Faraday::Response, body: response_body) }
134+
135+
it 'raises an error' do
136+
expect do
137+
images_module.parse_image_response(response, model: 'test-model')
138+
end.to raise_error(RubyLLM::Error, /Unexpected response format/)
139+
end
140+
end
141+
142+
context 'with empty images array' do
143+
let(:response_body) do
144+
{
145+
'choices' => [{
146+
'message' => { 'images' => [] }
147+
}]
148+
}
149+
end
150+
let(:response) { instance_double(Faraday::Response, body: response_body) }
151+
152+
it 'raises an error' do
153+
expect do
154+
images_module.parse_image_response(response, model: 'test-model')
155+
end.to raise_error(RubyLLM::Error, /Unexpected response format/)
156+
end
157+
end
158+
159+
context 'with invalid data URL format' do
160+
let(:response_body) do
161+
{
162+
'choices' => [{
163+
'message' => {
164+
'images' => [{
165+
'image_url' => { 'url' => 'data:invalid-format' }
166+
}]
167+
}
168+
}]
169+
}
170+
end
171+
let(:response) { instance_double(Faraday::Response, body: response_body) }
172+
173+
it 'raises an error for invalid data URL' do
174+
expect do
175+
images_module.parse_image_response(response, model: 'test-model')
176+
end.to raise_error(RubyLLM::Error, /Invalid data URL format/)
177+
end
178+
end
179+
end
180+
end

0 commit comments

Comments
 (0)