Skip to content
This repository was archived by the owner on Sep 18, 2025. It is now read-only.

Commit 9fec8df

Browse files
PhantomReactorPhantomReactor
authored
add support for images (#144)
1 parent 603a3e3 commit 9fec8df

File tree

30 files changed

+1326
-484
lines changed

30 files changed

+1326
-484
lines changed

‎go.mod‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ module github.com/opencode-ai/opencode
22

33
go 1.24.0
44

5-
toolchain go1.24.2
6-
75
require (
86
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
97
github.com/JohannesKaufmann/html-to-markdown v1.6.0
@@ -16,7 +14,6 @@ require (
1614
github.com/charmbracelet/bubbles v0.20.0
1715
github.com/charmbracelet/bubbletea v1.3.4
1816
github.com/charmbracelet/glamour v0.9.1
19-
github.com/charmbracelet/huh v0.6.0
2017
github.com/charmbracelet/lipgloss v1.1.0
2118
github.com/charmbracelet/x/ansi v0.8.0
2219
github.com/fsnotify/fsnotify v1.8.0
@@ -68,11 +65,10 @@ require (
6865
github.com/aymerick/douceur v0.2.0 // indirect
6966
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
7067
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
71-
github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect
7268
github.com/charmbracelet/x/term v0.2.1 // indirect
7369
github.com/davecgh/go-spew v1.1.1 // indirect
70+
github.com/disintegration/imaging v1.6.2
7471
github.com/dlclark/regexp2 v1.11.4 // indirect
75-
github.com/dustin/go-humanize v1.0.1 // indirect
7672
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
7773
github.com/felixge/httpsnoop v1.0.4 // indirect
7874
github.com/go-logr/logr v1.4.2 // indirect
@@ -85,13 +81,12 @@ require (
8581
github.com/gorilla/css v1.0.1 // indirect
8682
github.com/inconshreveable/mousetrap v1.1.0 // indirect
8783
github.com/kylelemons/godebug v1.1.0 // indirect
88-
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
84+
github.com/lucasb-eyer/go-colorful v1.2.0
8985
github.com/mattn/go-isatty v0.0.20 // indirect
9086
github.com/mattn/go-localereader v0.0.1 // indirect
9187
github.com/mattn/go-runewidth v0.0.16 // indirect
9288
github.com/mfridman/interpolate v0.0.2 // indirect
9389
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
94-
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
9590
github.com/muesli/cancelreader v0.2.2 // indirect
9691
github.com/ncruces/julianday v1.0.0 // indirect
9792
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
@@ -123,6 +118,7 @@ require (
123118
go.opentelemetry.io/otel/trace v1.35.0 // indirect
124119
go.uber.org/multierr v1.11.0 // indirect
125120
golang.org/x/crypto v0.37.0 // indirect
121+
golang.org/x/image v0.26.0 // indirect
126122
golang.org/x/net v0.39.0 // indirect
127123
golang.org/x/oauth2 v0.25.0 // indirect
128124
golang.org/x/sync v0.13.0 // indirect

‎go.sum‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,6 @@ github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4p
8282
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk=
8383
github.com/charmbracelet/glamour v0.9.1 h1:11dEfiGP8q1BEqvGoIjivuc2rBk+5qEXdPtaQ2WoiCM=
8484
github.com/charmbracelet/glamour v0.9.1/go.mod h1:+SHvIS8qnwhgTpVMiXwn7OfGomSqff1cHBCI8jLOetk=
85-
github.com/charmbracelet/huh v0.6.0 h1:mZM8VvZGuE0hoDXq6XLxRtgfWyTI3b2jZNKh0xWmax8=
86-
github.com/charmbracelet/huh v0.6.0/go.mod h1:GGNKeWCeNzKpEOh/OJD8WBwTQjV3prFAtQPpLv+AVwU=
8785
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
8886
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
8987
github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE=
@@ -92,14 +90,14 @@ github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0G
9290
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs=
9391
github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b h1:MnAMdlwSltxJyULnrYbkZpp4k58Co7Tah3ciKhSNo0Q=
9492
github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
95-
github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 h1:qko3AQ4gK1MTS/de7F5hPGx6/k1u0w4TeYmBFwzYVP4=
96-
github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0/go.mod h1:pBhA0ybfXv6hDjQUZ7hk1lVxBiUbupdw5R31yPUViVQ=
9793
github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
9894
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
9995
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
10096
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
10197
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
10298
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
99+
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
100+
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
103101
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
104102
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
105103
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
@@ -169,8 +167,6 @@ github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6B
169167
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
170168
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
171169
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
172-
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
173-
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
174170
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
175171
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
176172
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
@@ -283,6 +279,9 @@ golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
283279
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
284280
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
285281
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
282+
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
283+
golang.org/x/image v0.26.0 h1:4XjIFEZWQmCZi6Wv8BoxsDhRU3RVnLX04dToTDAEPlY=
284+
golang.org/x/image v0.26.0/go.mod h1:lcxbMFAovzpnJxzXS3nyL83K27tmqtKzIJpctK8YO5c=
286285
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
287286
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
288287
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

‎internal/llm/agent/agent.go‎

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func (e *AgentEvent) Response() message.Message {
3838
}
3939

4040
type Service interface {
41-
Run(ctx context.Context, sessionID string, content string) (<-chan AgentEvent, error)
41+
Run(ctx context.Context, sessionID string, content string, attachments ...message.Attachment) (<-chan AgentEvent, error)
4242
Cancel(sessionID string)
4343
IsSessionBusy(sessionID string) bool
4444
IsBusy() bool
@@ -117,23 +117,23 @@ func (a *agent) IsSessionBusy(sessionID string) bool {
117117
}
118118

119119
func (a *agent) generateTitle(ctx context.Context, sessionID string, content string) error {
120+
if content == "" {
121+
return nil
122+
}
120123
if a.titleProvider == nil {
121124
return nil
122125
}
123126
session, err := a.sessions.Get(ctx, sessionID)
124127
if err != nil {
125128
return err
126129
}
130+
parts := []message.ContentPart{message.TextContent{Text: content}}
127131
response, err := a.titleProvider.SendMessages(
128132
ctx,
129133
[]message.Message{
130134
{
131-
Role: message.User,
132-
Parts: []message.ContentPart{
133-
message.TextContent{
134-
Text: content,
135-
},
136-
},
135+
Role: message.User,
136+
Parts: parts,
137137
},
138138
},
139139
make([]tools.BaseTool, 0),
@@ -158,7 +158,10 @@ func (a *agent) err(err error) AgentEvent {
158158
}
159159
}
160160

161-
func (a *agent) Run(ctx context.Context, sessionID string, content string) (<-chan AgentEvent, error) {
161+
func (a *agent) Run(ctx context.Context, sessionID string, content string, attachments ...message.Attachment) (<-chan AgentEvent, error) {
162+
if !a.provider.Model().SupportsAttachments && attachments != nil {
163+
attachments = nil
164+
}
162165
events := make(chan AgentEvent)
163166
if a.IsSessionBusy(sessionID) {
164167
return nil, ErrSessionBusy
@@ -172,10 +175,13 @@ func (a *agent) Run(ctx context.Context, sessionID string, content string) (<-ch
172175
defer logging.RecoverPanic("agent.Run", func() {
173176
events <- a.err(fmt.Errorf("panic while running the agent"))
174177
})
175-
176-
result := a.processGeneration(genCtx, sessionID, content)
178+
var attachmentParts []message.ContentPart
179+
for _, attachment := range attachments {
180+
attachmentParts = append(attachmentParts, message.BinaryContent{Path: attachment.FilePath, MIMEType: attachment.MimeType, Data: attachment.Content})
181+
}
182+
result := a.processGeneration(genCtx, sessionID, content, attachmentParts)
177183
if result.Err() != nil && !errors.Is(result.Err(), ErrRequestCancelled) && !errors.Is(result.Err(), context.Canceled) {
178-
logging.ErrorPersist(fmt.Sprintf("Generation error for session %s: %v", sessionID, result))
184+
logging.ErrorPersist(result.Err().Error())
179185
}
180186
logging.Debug("Request completed", "sessionID", sessionID)
181187
a.activeRequests.Delete(sessionID)
@@ -186,7 +192,7 @@ func (a *agent) Run(ctx context.Context, sessionID string, content string) (<-ch
186192
return events, nil
187193
}
188194

189-
func (a *agent) processGeneration(ctx context.Context, sessionID, content string) AgentEvent {
195+
func (a *agent) processGeneration(ctx context.Context, sessionID, content string, attachmentParts []message.ContentPart) AgentEvent {
190196
// List existing messages; if none, start title generation asynchronously.
191197
msgs, err := a.messages.List(ctx, sessionID)
192198
if err != nil {
@@ -204,13 +210,13 @@ func (a *agent) processGeneration(ctx context.Context, sessionID, content string
204210
}()
205211
}
206212

207-
userMsg, err := a.createUserMessage(ctx, sessionID, content)
213+
userMsg, err := a.createUserMessage(ctx, sessionID, content, attachmentParts)
208214
if err != nil {
209215
return a.err(fmt.Errorf("failed to create user message: %w", err))
210216
}
211-
212217
// Append the new user message to the conversation history.
213218
msgHistory := append(msgs, userMsg)
219+
214220
for {
215221
// Check for cancellation before each iteration
216222
select {
@@ -240,12 +246,12 @@ func (a *agent) processGeneration(ctx context.Context, sessionID, content string
240246
}
241247
}
242248

243-
func (a *agent) createUserMessage(ctx context.Context, sessionID, content string) (message.Message, error) {
249+
func (a *agent) createUserMessage(ctx context.Context, sessionID, content string, attachmentParts []message.ContentPart) (message.Message, error) {
250+
parts := []message.ContentPart{message.TextContent{Text: content}}
251+
parts = append(parts, attachmentParts...)
244252
return a.messages.Create(ctx, sessionID, message.CreateMessageParams{
245-
Role: message.User,
246-
Parts: []message.ContentPart{
247-
message.TextContent{Text: content},
248-
},
253+
Role: message.User,
254+
Parts: parts,
249255
})
250256
}
251257

@@ -310,7 +316,6 @@ func (a *agent) streamAndHandleEvents(ctx context.Context, sessionID string, msg
310316
}
311317
continue
312318
}
313-
314319
toolResult, toolErr := tool.Run(ctx, tools.ToolCall{
315320
ID: toolCall.ID,
316321
Name: toolCall.Name,

‎internal/llm/models/anthropic.go‎

Lines changed: 56 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -14,64 +14,69 @@ const (
1414
// https://docs.anthropic.com/en/docs/about-claude/models/all-models
1515
var AnthropicModels = map[ModelID]Model{
1616
Claude35Sonnet: {
17-
ID: Claude35Sonnet,
18-
Name: "Claude 3.5 Sonnet",
19-
Provider: ProviderAnthropic,
20-
APIModel: "claude-3-5-sonnet-latest",
21-
CostPer1MIn: 3.0,
22-
CostPer1MInCached: 3.75,
23-
CostPer1MOutCached: 0.30,
24-
CostPer1MOut: 15.0,
25-
ContextWindow: 200000,
26-
DefaultMaxTokens: 5000,
17+
ID: Claude35Sonnet,
18+
Name: "Claude 3.5 Sonnet",
19+
Provider: ProviderAnthropic,
20+
APIModel: "claude-3-5-sonnet-latest",
21+
CostPer1MIn: 3.0,
22+
CostPer1MInCached: 3.75,
23+
CostPer1MOutCached: 0.30,
24+
CostPer1MOut: 15.0,
25+
ContextWindow: 200000,
26+
DefaultMaxTokens: 5000,
27+
SupportsAttachments: true,
2728
},
2829
Claude3Haiku: {
29-
ID: Claude3Haiku,
30-
Name: "Claude 3 Haiku",
31-
Provider: ProviderAnthropic,
32-
APIModel: "claude-3-haiku-20240307", // doesn't support "-latest"
33-
CostPer1MIn: 0.25,
34-
CostPer1MInCached: 0.30,
35-
CostPer1MOutCached: 0.03,
36-
CostPer1MOut: 1.25,
37-
ContextWindow: 200000,
38-
DefaultMaxTokens: 4096,
30+
ID: Claude3Haiku,
31+
Name: "Claude 3 Haiku",
32+
Provider: ProviderAnthropic,
33+
APIModel: "claude-3-haiku-20240307", // doesn't support "-latest"
34+
CostPer1MIn: 0.25,
35+
CostPer1MInCached: 0.30,
36+
CostPer1MOutCached: 0.03,
37+
CostPer1MOut: 1.25,
38+
ContextWindow: 200000,
39+
DefaultMaxTokens: 4096,
40+
SupportsAttachments: true,
3941
},
4042
Claude37Sonnet: {
41-
ID: Claude37Sonnet,
42-
Name: "Claude 3.7 Sonnet",
43-
Provider: ProviderAnthropic,
44-
APIModel: "claude-3-7-sonnet-latest",
45-
CostPer1MIn: 3.0,
46-
CostPer1MInCached: 3.75,
47-
CostPer1MOutCached: 0.30,
48-
CostPer1MOut: 15.0,
49-
ContextWindow: 200000,
50-
DefaultMaxTokens: 50000,
51-
CanReason: true,
43+
ID: Claude37Sonnet,
44+
Name: "Claude 3.7 Sonnet",
45+
Provider: ProviderAnthropic,
46+
APIModel: "claude-3-7-sonnet-latest",
47+
CostPer1MIn: 3.0,
48+
CostPer1MInCached: 3.75,
49+
CostPer1MOutCached: 0.30,
50+
CostPer1MOut: 15.0,
51+
ContextWindow: 200000,
52+
DefaultMaxTokens: 50000,
53+
CanReason: true,
54+
SupportsAttachments: true,
5255
},
5356
Claude35Haiku: {
54-
ID: Claude35Haiku,
55-
Name: "Claude 3.5 Haiku",
56-
Provider: ProviderAnthropic,
57-
APIModel: "claude-3-5-haiku-latest",
58-
CostPer1MIn: 0.80,
59-
CostPer1MInCached: 1.0,
60-
CostPer1MOutCached: 0.08,
61-
CostPer1MOut: 4.0,
62-
ContextWindow: 200000,
63-
DefaultMaxTokens: 4096,
57+
ID: Claude35Haiku,
58+
Name: "Claude 3.5 Haiku",
59+
Provider: ProviderAnthropic,
60+
APIModel: "claude-3-5-haiku-latest",
61+
CostPer1MIn: 0.80,
62+
CostPer1MInCached: 1.0,
63+
CostPer1MOutCached: 0.08,
64+
CostPer1MOut: 4.0,
65+
ContextWindow: 200000,
66+
DefaultMaxTokens: 4096,
67+
SupportsAttachments: true,
6468
},
6569
Claude3Opus: {
66-
ID: Claude3Opus,
67-
Name: "Claude 3 Opus",
68-
Provider: ProviderAnthropic,
69-
APIModel: "claude-3-opus-latest",
70-
CostPer1MIn: 15.0,
71-
CostPer1MInCached: 18.75,
72-
CostPer1MOutCached: 1.50,
73-
CostPer1MOut: 75.0,
74-
ContextWindow: 200000,
75-
DefaultMaxTokens: 4096,
70+
ID: Claude3Opus,
71+
Name: "Claude 3 Opus",
72+
Provider: ProviderAnthropic,
73+
APIModel: "claude-3-opus-latest",
74+
CostPer1MIn: 15.0,
75+
CostPer1MInCached: 18.75,
76+
CostPer1MOutCached: 1.50,
77+
CostPer1MOut: 75.0,
78+
ContextWindow: 200000,
79+
DefaultMaxTokens: 4096,
80+
SupportsAttachments: true,
7681
},
7782
}

0 commit comments

Comments
 (0)