Skip to content

Commit 2800463

Browse files
authored
fix: file annotations render (#149)
1 parent 9424b9e commit 2800463

File tree

8 files changed

+652
-41
lines changed

8 files changed

+652
-41
lines changed

src/components/BotMessage.test.tsx

Lines changed: 436 additions & 3 deletions
Large diffs are not rendered by default.

src/components/BotMessage.tsx

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,29 @@ import { toast } from 'sonner'
33
import { useState } from 'react'
44
import { MessageAvatar } from './MessageAvatar'
55
import type { AssistantStreamEvent } from '@/hooks/useStreamingChat'
6+
import type { AnnotatedFile } from '@/lib/utils/code-interpreter'
67
import { cn, formatTimestamp } from '@/lib/utils'
78
import { MarkdownContent } from '@/components/MarkdownContent'
89
import { copyToClipboard } from '@/lib/utils/clipboard'
910
import {
1011
createAnnotatedFileUrl,
1112
isImageFile,
13+
replaceSandboxUrls,
1214
} from '@/lib/utils/code-interpreter'
1315

14-
interface Message extends Omit<AssistantStreamEvent, 'type'> {
16+
export interface Message extends Omit<AssistantStreamEvent, 'type'> {
1517
timestamp: string
1618
status: string
1719
}
1820

1921
export interface BotMessageProps {
2022
message: Message
21-
fileAnnotations?: AssistantStreamEvent['fileAnnotations']
23+
fileAnnotations?: Array<AnnotatedFile>
2224
}
2325

2426
export function BotMessage({ message, fileAnnotations = [] }: BotMessageProps) {
2527
const [imageErrors, setImageErrors] = useState<Set<string>>(new Set())
28+
const processedContent = replaceSandboxUrls(message.content, fileAnnotations)
2629

2730
const handleCopy = async () => {
2831
const copied = await copyToClipboard(processedContent)
@@ -36,13 +39,8 @@ export function BotMessage({ message, fileAnnotations = [] }: BotMessageProps) {
3639
setImageErrors((prev) => new Set(prev).add(fileId))
3740
}
3841

39-
// Use message content directly; sandbox URL replacement is not needed
40-
const processedContent = message.content
41-
42-
// Separate image and non-image files
43-
// Only treat annotations with type 'file' or 'image' as downloadable
4442
const fileLikeAnnotations = fileAnnotations.filter(
45-
(annotation) => annotation.type === 'file' || annotation.type === 'image',
43+
(annotation) => annotation.type === 'container_file_citation',
4644
)
4745
const imageFiles = fileLikeAnnotations.filter((annotation) =>
4846
isImageFile(annotation.filename),
@@ -70,8 +68,11 @@ export function BotMessage({ message, fileAnnotations = [] }: BotMessageProps) {
7068
'rounded-2xl px-4 py-2 text-sm w-full bg-gray-100 text-gray-900 dark:bg-gray-800 dark:text-gray-100',
7169
)}
7270
>
73-
<div data-raw-markdown={message.content}>
74-
<MarkdownContent content={message.content} />
71+
<div data-raw-markdown={processedContent}>
72+
<MarkdownContent
73+
content={processedContent}
74+
fileAnnotations={fileAnnotations}
75+
/>
7576
</div>
7677

7778
{imageFiles.length > 0 && (
@@ -80,16 +81,18 @@ export function BotMessage({ message, fileAnnotations = [] }: BotMessageProps) {
8081
const hasError = imageErrors.has(annotation.file_id)
8182
const imageUrl = createAnnotatedFileUrl(annotation)
8283

83-
// Generate accessible alt text based on filename
8484
const getAltText = (filename: string) => {
85-
const name = filename.replace(/\.[^/.]+$/, '') // Remove extension
85+
const name = filename.replace(/\.[^/.]+$/, '')
8686
return `Generated visualization: ${name.replace(/_/g, ' ')}`
8787
}
8888

8989
return (
9090
<div key={annotation.file_id} className="relative group">
9191
{hasError ? (
92-
<div className="flex items-center justify-center p-4 bg-gray-50 dark:bg-gray-900 rounded border border-dashed border-gray-300 dark:border-gray-600">
92+
<div
93+
className="flex items-center justify-center p-4 bg-gray-50 dark:bg-gray-900 rounded border border-dashed border-gray-300 dark:border-gray-600"
94+
role="alert"
95+
>
9396
<div className="text-center">
9497
<p className="text-sm text-gray-500 dark:text-gray-400">
9598
Failed to load image
@@ -133,9 +136,9 @@ export function BotMessage({ message, fileAnnotations = [] }: BotMessageProps) {
133136
<p className="text-xs text-gray-500 dark:text-gray-400 font-medium">
134137
Attachments:
135138
</p>
136-
<div className="space-y-1">
139+
<ul className="space-y-1" aria-label="File attachments">
137140
{otherFiles.map((annotation) => (
138-
<div
141+
<li
139142
key={annotation.file_id}
140143
className="flex items-center justify-between p-2 bg-gray-50 dark:bg-gray-900 rounded border border-gray-200 dark:border-gray-700"
141144
>
@@ -149,10 +152,11 @@ export function BotMessage({ message, fileAnnotations = [] }: BotMessageProps) {
149152
>
150153
<Download className="h-3 w-3" />
151154
Download
155+
<span className="sr-only"> {annotation.filename}</span>
152156
</a>
153-
</div>
157+
</li>
154158
))}
155-
</div>
159+
</ul>
156160
</div>
157161
)}
158162
</div>

src/components/MarkdownContent.tsx

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import ReactMarkdown from 'react-markdown'
22
import remarkGfm from 'remark-gfm'
33
import { toast } from 'sonner'
4+
import type { AnnotatedFile } from '@/lib/utils/code-interpreter'
45
import { CodeBlock } from '@/components/CodeBlock'
56

67
type MarkdownContentProps = {
78
content: string
9+
fileAnnotations?: Array<AnnotatedFile>
810
}
911

1012
export function MarkdownContent({ content }: MarkdownContentProps) {

src/hooks/useStreamingChat.test.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -522,7 +522,7 @@ describe('useStreamingChat', () => {
522522
type: 'code_interpreter_file_annotation',
523523
itemId: 'file-123',
524524
annotation: {
525-
type: 'file',
525+
type: 'container_file_citation',
526526
container_id: 'container-123',
527527
file_id: 'file-456',
528528
filename: 'test.py',

src/hooks/useStreamingChat.ts

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,14 @@
11
import { useCallback, useEffect, useRef, useState } from 'react'
22
import { generateMessageId } from '../mcp/client'
3+
import type { AnnotatedFile } from '@/lib/utils/code-interpreter'
34
import { stopStreamProcessing } from '@/lib/utils/streaming'
45
import { getTimestamp } from '@/lib/utils/date'
56

67
export type AssistantStreamEvent = {
78
type: 'assistant'
89
id: string
910
content: string
10-
fileAnnotations?: Array<{
11-
type: string
12-
container_id: string
13-
file_id: string
14-
filename: string
15-
start_index?: number
16-
end_index?: number
17-
}>
11+
fileAnnotations?: Array<AnnotatedFile>
1812
}
1913

2014
export type ToolStreamEvent = {

src/lib/utils/code-interpreter.test.ts

Lines changed: 156 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,26 @@
1-
import { describe, expect, it } from 'vitest'
2-
import { createAnnotatedFileUrl, isImageFile } from './code-interpreter'
1+
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
2+
import {
3+
createAnnotatedFileUrl,
4+
isImageFile,
5+
replaceSandboxUrls,
6+
} from './code-interpreter'
37
import type { AnnotatedFile } from './code-interpreter'
48

9+
const originalLocation = global.location
10+
beforeEach(() => {
11+
Object.defineProperty(global, 'location', {
12+
value: { origin: 'http://localhost:3000' },
13+
writable: true,
14+
})
15+
})
16+
17+
afterEach(() => {
18+
Object.defineProperty(global, 'location', {
19+
value: originalLocation,
20+
writable: true,
21+
})
22+
})
23+
524
describe('isImageFile', () => {
625
it('returns true for supported image extensions', () => {
726
expect(isImageFile('photo.jpg')).toBe(true)
@@ -28,15 +47,143 @@ describe('createAnnotatedFileUrl', () => {
2847
filename: 'test.png',
2948
}
3049

31-
it('returns absolute URL with correct params (browser)', () => {
32-
Object.defineProperty(globalThis, 'location', {
33-
value: { origin: 'https://example.com' },
34-
writable: true,
35-
})
36-
50+
it('returns absolute URL with correct params including filename (browser)', () => {
3751
const url = createAnnotatedFileUrl(file)
3852
expect(url).toBe(
39-
'https://example.com/api/container-file?containerId=c1&fileId=f1',
53+
'http://localhost:3000/api/container-file?containerId=c1&fileId=f1&filename=test.png',
54+
)
55+
})
56+
57+
it('handles filenames with special characters', () => {
58+
const fileWithSpecialChars: AnnotatedFile = {
59+
type: 'container_file_citation',
60+
container_id: 'c2',
61+
file_id: 'f2',
62+
filename: 'my file with spaces.py',
63+
}
64+
const url = createAnnotatedFileUrl(fileWithSpecialChars)
65+
expect(url).toBe(
66+
'http://localhost:3000/api/container-file?containerId=c2&fileId=f2&filename=my+file+with+spaces.py',
4067
)
4168
})
69+
70+
it('handles filenames with special URL characters', () => {
71+
const fileWithUrlChars: AnnotatedFile = {
72+
type: 'container_file_citation',
73+
container_id: 'c3',
74+
file_id: 'f3',
75+
filename: 'file&with=special?chars.txt',
76+
}
77+
const url = createAnnotatedFileUrl(fileWithUrlChars)
78+
expect(url).toBe(
79+
'http://localhost:3000/api/container-file?containerId=c3&fileId=f3&filename=file%26with%3Dspecial%3Fchars.txt',
80+
)
81+
})
82+
})
83+
84+
describe('replaceSandboxUrls', () => {
85+
const mockAnnotations: Array<AnnotatedFile> = [
86+
{
87+
type: 'container_file_citation',
88+
container_id: 'container-123',
89+
file_id: 'file-456',
90+
filename: 'example.py',
91+
},
92+
{
93+
type: 'container_file_citation',
94+
container_id: 'container-789',
95+
file_id: 'file-101',
96+
filename: 'data.csv',
97+
},
98+
{
99+
type: 'other_type',
100+
container_id: 'container-999',
101+
file_id: 'file-999',
102+
filename: 'ignored.txt',
103+
},
104+
]
105+
106+
it('should return original content when no file annotations are provided', () => {
107+
const content = 'Check out sandbox:/mnt/data/example.py for details'
108+
const result = replaceSandboxUrls(content, [])
109+
expect(result).toBe(content)
110+
})
111+
112+
it('should replace sandbox URLs with container-file URLs', () => {
113+
const content = 'Check out sandbox:/mnt/data/example.py for details'
114+
const result = replaceSandboxUrls(content, mockAnnotations)
115+
116+
const expectedUrl = createAnnotatedFileUrl(mockAnnotations[0])
117+
expect(result).toBe(`Check out ${expectedUrl} for details`)
118+
})
119+
120+
it('should handle multiple sandbox URLs in the same content', () => {
121+
const content =
122+
'Files: sandbox:/mnt/data/example.py and sandbox:/mnt/data/data.csv'
123+
const result = replaceSandboxUrls(content, mockAnnotations)
124+
125+
const expectedUrl1 = createAnnotatedFileUrl(mockAnnotations[0])
126+
const expectedUrl2 = createAnnotatedFileUrl(mockAnnotations[1])
127+
expect(result).toBe(`Files: ${expectedUrl1} and ${expectedUrl2}`)
128+
})
129+
130+
it('should preserve sandbox URLs that do not have matching annotations', () => {
131+
const content = 'Check out sandbox:/mnt/data/unknown.py'
132+
const result = replaceSandboxUrls(content, mockAnnotations)
133+
expect(result).toBe(content)
134+
})
135+
136+
it('should handle mixed content with some matching and some non-matching URLs', () => {
137+
const content =
138+
'Files: sandbox:/mnt/data/example.py and sandbox:/mnt/data/unknown.py'
139+
const result = replaceSandboxUrls(content, mockAnnotations)
140+
141+
const expectedUrl = createAnnotatedFileUrl(mockAnnotations[0])
142+
expect(result).toBe(
143+
`Files: ${expectedUrl} and sandbox:/mnt/data/unknown.py`,
144+
)
145+
})
146+
147+
it('should ignore annotations that are not container_file_citation type', () => {
148+
const content = 'Check out sandbox:/mnt/data/ignored.txt'
149+
const result = replaceSandboxUrls(content, mockAnnotations)
150+
expect(result).toBe(content)
151+
})
152+
153+
it('should handle URLs with different formats (parentheses, brackets)', () => {
154+
const content =
155+
'See (sandbox:/mnt/data/example.py) and [sandbox:/mnt/data/data.csv]'
156+
const result = replaceSandboxUrls(content, mockAnnotations)
157+
158+
const expectedUrl1 = createAnnotatedFileUrl(mockAnnotations[0])
159+
const expectedUrl2 = createAnnotatedFileUrl(mockAnnotations[1])
160+
expect(result).toBe(`See (${expectedUrl1}) and [${expectedUrl2}]`)
161+
})
162+
163+
it('should handle URLs with whitespace after them', () => {
164+
const content =
165+
'Check sandbox:/mnt/data/example.py and sandbox:/mnt/data/data.csv '
166+
const result = replaceSandboxUrls(content, mockAnnotations)
167+
168+
const expectedUrl1 = createAnnotatedFileUrl(mockAnnotations[0])
169+
const expectedUrl2 = createAnnotatedFileUrl(mockAnnotations[1])
170+
expect(result).toBe(`Check ${expectedUrl1} and ${expectedUrl2} `)
171+
})
172+
173+
it('should handle empty content', () => {
174+
const result = replaceSandboxUrls('', mockAnnotations)
175+
expect(result).toBe('')
176+
})
177+
178+
it('should handle content with no sandbox URLs', () => {
179+
const content = 'This is just regular text with no URLs'
180+
const result = replaceSandboxUrls(content, mockAnnotations)
181+
expect(result).toBe(content)
182+
})
183+
184+
it('should handle malformed sandbox URLs gracefully', () => {
185+
const content = 'Check sandbox:/mnt/data/ and sandbox:/invalid'
186+
const result = replaceSandboxUrls(content, mockAnnotations)
187+
expect(result).toBe(content)
188+
})
42189
})

src/lib/utils/code-interpreter.ts

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
// Utility functions for file handling (shared between components)
2-
31
export interface AnnotatedFile {
42
type: string
53
container_id: string
@@ -19,5 +17,36 @@ export function createAnnotatedFileUrl(file: AnnotatedFile): string {
1917
const url = new URL('/api/container-file', location.origin)
2018
url.searchParams.set('containerId', file.container_id)
2119
url.searchParams.set('fileId', file.file_id)
20+
url.searchParams.set('filename', file.filename)
2221
return url.toString()
2322
}
23+
24+
/**
25+
* Replaces sandbox URLs in markdown content with /api/container-file URLs using file annotations.
26+
* Sandbox URLs are only valid within the code interpreter container, so they need to be converted
27+
* to accessible URLs for the frontend.
28+
*/
29+
export function replaceSandboxUrls(
30+
markdownContent: string,
31+
fileAnnotations: Array<AnnotatedFile> = [],
32+
): string {
33+
if (!fileAnnotations.length) return markdownContent
34+
35+
const fileMap = new Map<string, AnnotatedFile>()
36+
fileAnnotations.forEach((annotation) => {
37+
if (annotation.type === 'container_file_citation') {
38+
fileMap.set(annotation.filename, annotation)
39+
}
40+
})
41+
42+
return markdownContent.replace(
43+
/sandbox:\/mnt\/data\/([^\s)\]]+)/g,
44+
(match, filename) => {
45+
const annotation = fileMap.get(filename)
46+
if (annotation) {
47+
return createAnnotatedFileUrl(annotation)
48+
}
49+
return match
50+
},
51+
)
52+
}

src/routes/api/container-file.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export const ServerRoute = createServerFileRoute('/api/container-file').methods(
2727
const queryParams = {
2828
containerId: url.searchParams.get('containerId'),
2929
fileId: url.searchParams.get('fileId'),
30+
filename: url.searchParams.get('filename'),
3031
}
3132

3233
const validationResult = containerFileQuerySchema.safeParse(queryParams)
@@ -81,7 +82,8 @@ export const ServerRoute = createServerFileRoute('/api/container-file').methods(
8182
// Try to get file metadata using the regular Files API
8283
// This API returns metadata only (filename, size, etc.) - NOT the file content
8384
// Note: Container files may not be accessible via this API, so we fallback gracefully
84-
const filename = await getFilenameFromMetadata(fileId)
85+
const filename =
86+
queryParams.filename ?? (await getFilenameFromMetadata(fileId))
8587
const contentType = mime.getType(filename) || 'application/octet-stream'
8688

8789
return new Response(arrayBuffer, {

0 commit comments

Comments
 (0)