Skip to content

Commit 49baf40

Browse files
slinjhuslinjhu
committed
Added embedding for images
1 parent e582c17 commit 49baf40

File tree

3 files changed

+232
-0
lines changed

3 files changed

+232
-0
lines changed

‎smoosense-gui/src/app/example/emb-images/page.tsx‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
'use client'
2+
3+
import { Suspense } from 'react'
4+
import ExampleEmbImages from '@/lib/example/ExampleEmbImages'
5+
import ExamplePageLayout from '@/components/layout/ExamplePageLayout'
6+
7+
function EmbImagesContent() {
8+
return <ExampleEmbImages />
9+
}
10+
11+
export default function ExampleEmbImagesPage() {
12+
return (
13+
<ExamplePageLayout title="Image Embedding UMAP Example">
14+
<Suspense fallback={<div className="h-full flex items-center justify-center">Loading...</div>}>
15+
<EmbImagesContent />
16+
</Suspense>
17+
</ExamplePageLayout>
18+
)
19+
}

‎smoosense-gui/src/components/home/ExampleVisualization.tsx‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,16 @@ const EXAMPLES: ExampleCard[] = [
3131
href: '/example/audiomelspectrogram',
3232
tags: ['audio']
3333
},
34+
{
35+
title: 'Audio Embedding UMAP',
36+
href: '/example/emb-audio',
37+
tags: ['audio', 'embedding', 'umap']
38+
},
39+
{
40+
title: 'Image Embedding UMAP',
41+
href: '/example/emb-images',
42+
tags: ['image', 'embedding', 'umap']
43+
},
3444
{
3545
title: '3D Objects',
3646
tags: ['3d'],

‎smoosense-gui/src/lib/example/ExampleEmbImages.tsx‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
'use client'
2+
3+
import { useState, useCallback, useMemo, useEffect } from 'react'
4+
import { useAppSelector } from '@/lib/hooks'
5+
import { ResizablePanels } from '@/components/ui/resizable-panels'
6+
import Umap2DScatterPlot from '@/components/emb/Umap2DScatterPlot'
7+
import GalleryItem from '@/components/gallery/GalleryItem'
8+
import { RenderType } from '@/lib/utils/agGridCellRenderers'
9+
10+
// Data URL for image embeddings
11+
const DATA_URL = 'https://cdn.smoosense.ai/demo/photos/umap.jsonl'
12+
13+
// Base URL for images
14+
const IMAGES_BASE_URL = 'https://cdn.smoosense.ai/demo/photos'
15+
16+
interface ImageDataItem {
17+
image_path: string
18+
x: number
19+
y: number
20+
cluster_id: number
21+
}
22+
23+
interface UmapData {
24+
x: number[]
25+
y: number[]
26+
imagePaths: string[]
27+
clusterIds: number[]
28+
}
29+
30+
// Convert image_path to full URL
31+
function imagePathToUrl(imagePath: string): string {
32+
// Remove leading "./" prefix if present, then add base URL
33+
const cleanPath = imagePath.replace(/^\.\//, '')
34+
return `${IMAGES_BASE_URL}/${cleanPath}`
35+
}
36+
37+
// Parse JSONL data
38+
async function loadData(): Promise<UmapData> {
39+
const response = await fetch(DATA_URL)
40+
const text = await response.text()
41+
const lines = text.trim().split('\n')
42+
const items: ImageDataItem[] = lines.map(line => JSON.parse(line))
43+
44+
return {
45+
x: items.map(item => item.x),
46+
y: items.map(item => item.y),
47+
imagePaths: items.map(item => item.image_path),
48+
clusterIds: items.map(item => item.cluster_id)
49+
}
50+
}
51+
52+
interface SelectedItem {
53+
index: number
54+
imagePath: string
55+
imageUrl: string
56+
clusterId: number
57+
}
58+
59+
60+
export default function ExampleEmbImages() {
61+
const [selectedIndices, setSelectedIndices] = useState<number[]>([])
62+
const [umapData, setUmapData] = useState<UmapData | null>(null)
63+
const [isLoading, setIsLoading] = useState(true)
64+
const [error, setError] = useState<string | null>(null)
65+
const galleryItemWidth = useAppSelector((state) => state.ui.galleryItemWidth)
66+
67+
// Load data on mount
68+
useEffect(() => {
69+
loadData()
70+
.then(data => {
71+
setUmapData(data)
72+
setIsLoading(false)
73+
})
74+
.catch(err => {
75+
setError(err.message || 'Failed to load data')
76+
setIsLoading(false)
77+
})
78+
}, [])
79+
80+
const handleSelectionChange = useCallback((indices: number[]) => {
81+
setSelectedIndices(indices)
82+
}, [])
83+
84+
// Get selected items for gallery
85+
const selectedItems: SelectedItem[] = useMemo(() => {
86+
if (!umapData) return []
87+
return selectedIndices.map(idx => ({
88+
index: idx,
89+
imagePath: umapData.imagePaths[idx],
90+
imageUrl: imagePathToUrl(umapData.imagePaths[idx]),
91+
clusterId: umapData.clusterIds[idx]
92+
}))
93+
}, [selectedIndices, umapData])
94+
95+
// Prepare visual values (image URLs) for hover tooltip
96+
const visualValues = useMemo(() => {
97+
if (!umapData) return []
98+
return umapData.imagePaths.map(imagePathToUrl)
99+
}, [umapData])
100+
101+
// Prepare color values from cluster IDs
102+
const colorValues = useMemo(() => {
103+
if (!umapData) return []
104+
return umapData.clusterIds
105+
}, [umapData])
106+
107+
// Prepare caption values for hover tooltip
108+
const captionValues = useMemo(() => {
109+
if (!umapData) return []
110+
return umapData.clusterIds.map(id => `cluster ${id}`)
111+
}, [umapData])
112+
113+
if (isLoading) {
114+
return (
115+
<div className="h-full w-full flex items-center justify-center">
116+
<div className="text-muted-foreground animate-pulse">Loading image data...</div>
117+
</div>
118+
)
119+
}
120+
121+
if (error) {
122+
return (
123+
<div className="h-full w-full flex items-center justify-center">
124+
<div className="text-destructive">Error: {error}</div>
125+
</div>
126+
)
127+
}
128+
129+
if (!umapData) {
130+
return null
131+
}
132+
133+
return (
134+
<div className="h-full w-full">
135+
<ResizablePanels
136+
direction="horizontal"
137+
defaultSizes={[60, 40]}
138+
minSize={20}
139+
maxSize={80}
140+
>
141+
{/* Left: Scatter Plot */}
142+
<div className="h-full bg-background flex flex-col">
143+
<div className="flex-1 min-h-0">
144+
<Umap2DScatterPlot
145+
x={umapData.x}
146+
y={umapData.y}
147+
visualValues={visualValues}
148+
captionValues={captionValues}
149+
colorValues={colorValues}
150+
visualRenderType={RenderType.ImageUrl}
151+
onSelectionChange={handleSelectionChange}
152+
/>
153+
</div>
154+
<div className="flex-shrink-0 px-4 py-2 border-t text-xs text-muted-foreground">
155+
{umapData.x.length} points, colored by cluster_id
156+
</div>
157+
</div>
158+
159+
{/* Right: Gallery */}
160+
<div className="h-full flex flex-col bg-muted/10 border-l">
161+
<div className="flex-shrink-0 px-4 py-3 border-b bg-background">
162+
<h3 className="font-medium">Selected Images</h3>
163+
<p className="text-xs text-muted-foreground mt-1">
164+
{selectedItems.length === 0
165+
? 'Click or lasso select points to view images'
166+
: `${selectedItems.length} item${selectedItems.length !== 1 ? 's' : ''} selected`}
167+
</p>
168+
</div>
169+
170+
<div className="flex-1 overflow-auto p-4">
171+
{selectedItems.length === 0 ? (
172+
<div className="h-full flex items-center justify-center">
173+
<div className="text-muted-foreground text-center">
174+
<p className="text-sm">Click or lasso select points</p>
175+
<p className="text-xs mt-1">to view images here</p>
176+
</div>
177+
</div>
178+
) : (
179+
<div
180+
className="grid gap-4 justify-items-center"
181+
style={{
182+
gridTemplateColumns: `repeat(auto-fill, ${galleryItemWidth}px)`
183+
}}
184+
>
185+
{selectedItems.map((item) => (
186+
<GalleryItem
187+
key={item.index}
188+
row={{}}
189+
index={item.index}
190+
visualValue={item.imageUrl}
191+
captionValue={`cluster ${item.clusterId}`}
192+
renderType={RenderType.ImageUrl}
193+
onClick={null}
194+
/>
195+
))}
196+
</div>
197+
)}
198+
</div>
199+
</div>
200+
</ResizablePanels>
201+
</div>
202+
)
203+
}

0 commit comments

Comments
 (0)