Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added data/media/audio.mp3
Binary file not shown.
Binary file added data/media/example.pdf
Binary file not shown.
Binary file added data/media/image.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added data/media/table.parquet
Binary file not shown.
Binary file added data/media/video.mp4
Binary file not shown.
20 changes: 20 additions & 0 deletions docs/media-urls.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,26 @@ For **every cell in every row**:
[useProcessedRowData.ts](../smoosense-gui/src/lib/hooks/useProcessedRowData.ts)
- Returns the processed data with all media URLs resolved

## Query Tab Processing

The Query tab also processes media URLs in query results:

**Location**: `SqlQueryPanel.tsx` @ [SqlQueryPanel.tsx](../smoosense-gui/src/components/sql/SqlQueryPanel.tsx)

**Process**:
1. Execute SQL query via `/api/query` endpoint
2. Receive results as `{column_names, rows}`
3. Transform rows using `_.zipObject(column_names, rows)`
4. **Process media URLs** using same logic as Samples tab:
- For each cell, check `needToResolveMediaUrl(value)`
- If true, call `resolveAssetUrl(value, tablePath, baseUrl)`
5. Render in `BasicAGTable` with auto-detected cell renderers

**Media Rendering**:
- Images displayed in `ImageCellRenderer`
- Videos displayed in `VideoCellRenderer`
- Audio displayed in `AudioCellRenderer`
- Cell renderers auto-detected via `inferRenderTypeFromData()`

## Backend Integration

Expand Down
13 changes: 6 additions & 7 deletions smoosense-gui/src/components/common/BasicAGTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,12 @@ export default function BasicAGTable({ data, className = '', onGridReady, colDef
const baseColumnDefs = inferColumnDefinitions(data)

// Apply overrides if provided
const columnDefs = colDefOverrides
? baseColumnDefs.map(colDef => {
const fieldName = colDef.field
const overrides = fieldName ? colDefOverrides[fieldName] : undefined
return overrides ? { ...colDef, ...overrides } : colDef
})
: baseColumnDefs
const columnDefs = baseColumnDefs.map(colDef => {
const fieldName = colDef.field
const overrides = colDefOverrides && fieldName ? colDefOverrides[fieldName] : undefined

return overrides ? { ...colDef, ...overrides } : colDef
})

// Construct final grid options
const finalGridOptions: GridOptions = {
Expand Down
4 changes: 1 addition & 3 deletions smoosense-gui/src/components/common/ImageBlock.tsx
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
'use client'

import { useAppSelector } from '@/lib/hooks'
import { proxyedUrl } from '@/lib/utils/urlUtils'

interface ImageBlockProps {
src: string
Expand All @@ -19,7 +18,6 @@ export default function ImageBlock({
neverFitCover = false
}: ImageBlockProps) {
const cropMediaToFitCover = useAppSelector((state) => state.ui.cropMediaToFitCover)
const imageUrl = proxyedUrl(src)

const finalClassName = `${className} ${(cropMediaToFitCover && !neverFitCover) ? 'object-cover' : 'object-contain'}`.trim()

Expand All @@ -31,7 +29,7 @@ export default function ImageBlock({

return (
<img
src={imageUrl}
src={src}
alt={alt}
className={finalClassName}
style={style}
Expand Down
5 changes: 1 addition & 4 deletions smoosense-gui/src/components/common/VideoPlayer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import { memo, useRef, useEffect, useState } from 'react'
import { useAppSelector } from '@/lib/hooks'
import { proxyedUrl } from '@/lib/utils/urlUtils'
import { Play } from 'lucide-react'

export interface VideoPlayerProps {
Expand All @@ -27,8 +26,6 @@ const VideoPlayer = memo(function VideoPlayer({
const autoPlayAllVideos = useAppSelector((state) => state.ui.autoPlayAllVideos)
const cropMediaToFitCover = useAppSelector((state) => state.ui.cropMediaToFitCover)

const videoUrl = proxyedUrl(src)

// Apply crop fit logic - always take full width and height
const finalClassName = cropMediaToFitCover
? `w-full h-full object-cover ${className || ''}`.trim()
Expand Down Expand Up @@ -108,7 +105,7 @@ const VideoPlayer = memo(function VideoPlayer({
>
<video
ref={videoRef}
src={videoUrl}
src={src}
className={finalClassName}
muted={galleryVideoMuted}
autoPlay={shouldAutoPlay}
Expand Down
5 changes: 2 additions & 3 deletions smoosense-gui/src/components/gallery/Gallery.tsx
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
'use client'

import { useAppSelector, useAppDispatch } from '@/lib/hooks'
import { useRenderType } from '@/lib/hooks'
import { useProcessedRowData } from '@/lib/hooks/useProcessedRowData'
import { useRenderType, useRowData } from '@/lib/hooks'
import { setJustClickedRowId } from '@/lib/features/viewing/viewingSlice'
import { handPickRow } from '@/lib/features/handPickedRows/handPickedRowsSlice'
import { isVisualType } from '@/lib/utils/renderTypeUtils'
Expand All @@ -12,7 +11,7 @@ import GalleryItem from './GalleryItem'

export default function Gallery() {
const dispatch = useAppDispatch()
const { data: rowData } = useProcessedRowData()
const { data: rowData } = useRowData()
const renderTypeColumns = useRenderType()
const columnForGalleryVisual = useAppSelector((state) => state.ui.columnForGalleryVisual)
const columnForGalleryCaption = useAppSelector((state) => state.ui.columnForGalleryCaption)
Expand Down
23 changes: 14 additions & 9 deletions smoosense-gui/src/components/gallery/GalleryItemVisual.tsx
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use client'

import { RenderType } from '@/lib/utils/agGridCellRenderers'
import { proxyedUrl } from '@/lib/utils/urlUtils'
import { mayResolveUrl } from '@/lib/utils/mediaUrlUtils'
import { parseBbox, buildBboxVizUrl } from '@/lib/utils/bboxUtils'
import ImageBlock from '@/components/common/ImageBlock'
import ImageMask from '@/components/viz/ImageMask'
Expand All @@ -24,48 +24,53 @@ export default function GalleryItemVisual({
index,
galleryItemHeight
}: GalleryItemVisualProps) {
const tablePath = useAppSelector((state) => state.ui.tablePath)
const baseUrl = useAppSelector((state) => state.ui.baseUrl)

// Resolve URLs once to avoid repeated function calls
const resolvedVisualUrl = mayResolveUrl({ value: visualValue, tablePath, baseUrl })
const resolvedImageUrl = mayResolveUrl({ value: row.image_url, tablePath, baseUrl })

return (
<div
className="relative overflow-hidden"
style={{ height: `${galleryItemHeight}px` }}
>
{renderType === RenderType.ImageUrl && (
<ImageBlock
src={String(visualValue)}
src={resolvedVisualUrl}
alt={`Row ${index + 1}`}
className="w-full h-full"
/>
)}

{renderType === RenderType.ImageMask && (
<ImageMask
image_url={String(row.image_url)}
mask_url={String(visualValue)}
image_url={resolvedImageUrl}
mask_url={resolvedVisualUrl}
alt={`Row ${index + 1}`}
/>
)}

{renderType === RenderType.VideoUrl && (
<GalleryVideoItem visualValue={String(visualValue)} />
<GalleryVideoItem visualValue={resolvedVisualUrl} />
)}

{renderType === RenderType.AudioUrl && (
<div className="w-full h-full flex items-center justify-center">
<AudioMiniMelSpectrogram audioUrl={proxyedUrl(String(visualValue))} height={galleryItemHeight} allowPopOver={true} />
<AudioMiniMelSpectrogram audioUrl={resolvedVisualUrl} height={galleryItemHeight} allowPopOver={true} />
</div>
)}

{renderType === RenderType.Bbox && (() => {
const bbox = parseBbox(visualValue)
const imageUrl = row.image_url

if (!bbox || !imageUrl || typeof imageUrl !== 'string' || !baseUrl) {
if (!bbox || !imageUrl || typeof imageUrl !== 'string' || !baseUrl || !tablePath) {
return null
}

const vizUrl = buildBboxVizUrl(imageUrl, [bbox], baseUrl)
const vizUrl = buildBboxVizUrl(imageUrl, [bbox], tablePath, baseUrl)

return (
<iframe
Expand All @@ -87,7 +92,7 @@ export default function GalleryItemVisual({

return (
<iframe
src={proxyedUrl(iframeUrl)}
src={iframeUrl}
className="w-full h-full border-0"
title={`Row ${index + 1}`}
style={{ backgroundColor: 'transparent' }}
Expand Down
5 changes: 2 additions & 3 deletions smoosense-gui/src/components/table/MainTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@ import { useRef, useCallback, useImperativeHandle, forwardRef, useMemo, memo, us
import { shallowEqual } from 'react-redux'
import { AgGridReact } from 'ag-grid-react'
import { GridReadyEvent, GridApi, ColumnResizedEvent, ColumnVisibleEvent, RowClickedEvent, SortChangedEvent } from 'ag-grid-community'
import { useAGGridTheme, useAg, useRenderType } from '@/lib/hooks'
import { useAGGridTheme, useAg, useRenderType, useRowData } from '@/lib/hooks'
import { useAppSelector, useAppDispatch } from '@/lib/hooks'
import { useProcessedRowData } from '@/lib/hooks/useProcessedRowData'
import { AllCommunityModule, ModuleRegistry } from 'ag-grid-community'
import { expandColDef, RenderType } from '@/lib/utils/agGridCellRenderers'
import { AlertCircle } from 'lucide-react'
Expand Down Expand Up @@ -52,7 +51,7 @@ const MainTable = memo(forwardRef<MainTableRef, object>((_props, ref) => {
const gridApiRef = useRef<GridApi | null>(null)

// Always call hooks - but handle null tablePath gracefully in the hooks
const { data, error: dataError } = useProcessedRowData()
const { data, error: dataError } = useRowData()
const { ag: baseColumnDefs } = useAg()
const renderTypeColumns = useRenderType()

Expand Down
49 changes: 11 additions & 38 deletions smoosense-gui/src/components/table/RowDetails.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,9 @@ import JsonBox from '@/components/ui/JsonBox'
import { RenderType } from '@/lib/utils/agGridCellRenderers'
import { List, X } from 'lucide-react'
import { isNil } from 'lodash'
import { useRenderType } from '@/lib/hooks'
import { useProcessedRowData } from '@/lib/hooks/useProcessedRowData'
import { useRenderType, useRowData } from '@/lib/hooks'
import AutoLink from '@/components/common/AutoLink'
import { setShowRowDetailsPanel } from '@/lib/features/ui/uiSlice'
import RichAudioPlayer from '@/components/audio/RichAudioPlayer'
import { proxyedUrl } from '@/lib/utils/urlUtils'

interface RowDetailsWrapperProps {
children: React.ReactNode
Expand Down Expand Up @@ -51,10 +48,13 @@ function RowDetailsWrapper({ children }: RowDetailsWrapperProps) {



function renderValueByType(value: unknown, renderType: RenderType): React.ReactNode {
function renderValueByType(
value: unknown,
renderType: RenderType
): React.ReactNode {
if (value === null) return <span className="text-muted-foreground italic">null</span>
if (value === undefined) return <span className="text-muted-foreground italic">undefined</span>

switch (renderType) {
case RenderType.Json:
if (typeof value === 'object' && !isNil(value)) {
Expand All @@ -71,55 +71,28 @@ function renderValueByType(value: unknown, renderType: RenderType): React.ReactN
}
}
return <span className="text-sm font-mono">{String(value)}</span>

case RenderType.ImageUrl:
if (typeof value === 'string') {
return (
<div>
<AutoLink url={value} className="text-xs font-mono" />
</div>
)
}
return <span className="text-sm font-mono">{String(value)}</span>


case RenderType.IFrame:
if (typeof value === 'string') {
return <AutoLink url={value} className="text-xs font-mono" />
}
return <span className="text-sm font-mono">{String(value)}</span>

case RenderType.VideoUrl:
if (typeof value === 'string') {
return (
<div>
<AutoLink url={value} className="text-xs font-mono" />
</div>
)
}
return <span className="text-sm font-mono">{String(value)}</span>

case RenderType.AudioUrl:
if (typeof value === 'string') {
const audioUrl = proxyedUrl(value)
return <RichAudioPlayer audioUrl={audioUrl} autoPlay={false} />
}
return <span className="text-sm font-mono">{String(value)}</span>

case RenderType.Boolean:
return <span className={`font-medium`}>{String(value)}</span>

case RenderType.Number:
return <span className="font-mono">{String(value)}</span>

default:
return <span className="text-sm font-mono">{String(value)}</span>
}
}

export default function RowDetails() {
// Get data from Redux store (using processed data that includes derived columns)
// Get data from Redux store
const justClickedRowId = useAppSelector((state) => state.viewing.justClickedRowId)
const { data: rowData } = useProcessedRowData()
const { data: rowData } = useRowData()
const columnDefs = useAppSelector((state) => state.ag.columnDefs)
const renderTypeColumns = useRenderType()

Expand Down
1 change: 1 addition & 0 deletions smoosense-gui/src/components/ui/CellPopover.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ export default function CellPopover({
<div className='flex flex-col h-full min-h-0'>
<CellPopoverContentHeader
url={url}
title={copyValue || undefined}
isExpanded={isExpanded}
onToggleExpand={handleToggleExpand}
onClose={handleClose}
Expand Down
Loading