Skip to content

Commit eb1e90b

Browse files
waleedlatif1waleedlatif1
authored
improvement(search): added more granular logs search, added logs export, improved overall search experience (#1378)
* improvement(search): added more granular logs search, added logs export, improved overall search experience * updated tests
1 parent 3905d1c commit eb1e90b

File tree

9 files changed

+584
-82
lines changed

9 files changed

+584
-82
lines changed

‎apps/sim/app/api/logs/export/route.ts‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
import { db } from '@sim/db'
2+
import { permissions, workflow, workflowExecutionLogs } from '@sim/db/schema'
3+
import { and, desc, eq, gte, inArray, lte, type SQL, sql } from 'drizzle-orm'
4+
import { type NextRequest, NextResponse } from 'next/server'
5+
import { z } from 'zod'
6+
import { getSession } from '@/lib/auth'
7+
import { createLogger } from '@/lib/logs/console/logger'
8+
9+
const logger = createLogger('LogsExportAPI')
10+
11+
export const revalidate = 0
12+
13+
const ExportParamsSchema = z.object({
14+
level: z.string().optional(),
15+
workflowIds: z.string().optional(),
16+
folderIds: z.string().optional(),
17+
triggers: z.string().optional(),
18+
startDate: z.string().optional(),
19+
endDate: z.string().optional(),
20+
search: z.string().optional(),
21+
workflowName: z.string().optional(),
22+
folderName: z.string().optional(),
23+
workspaceId: z.string(),
24+
})
25+
26+
function escapeCsv(value: any): string {
27+
if (value === null || value === undefined) return ''
28+
const str = String(value)
29+
if (/[",\n]/.test(str)) {
30+
return `"${str.replace(/"/g, '""')}"`
31+
}
32+
return str
33+
}
34+
35+
export async function GET(request: NextRequest) {
36+
try {
37+
const session = await getSession()
38+
if (!session?.user?.id) {
39+
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
40+
}
41+
42+
const userId = session.user.id
43+
const { searchParams } = new URL(request.url)
44+
const params = ExportParamsSchema.parse(Object.fromEntries(searchParams.entries()))
45+
46+
const selectColumns = {
47+
id: workflowExecutionLogs.id,
48+
workflowId: workflowExecutionLogs.workflowId,
49+
executionId: workflowExecutionLogs.executionId,
50+
level: workflowExecutionLogs.level,
51+
trigger: workflowExecutionLogs.trigger,
52+
startedAt: workflowExecutionLogs.startedAt,
53+
endedAt: workflowExecutionLogs.endedAt,
54+
totalDurationMs: workflowExecutionLogs.totalDurationMs,
55+
cost: workflowExecutionLogs.cost,
56+
executionData: workflowExecutionLogs.executionData,
57+
workflowName: workflow.name,
58+
}
59+
60+
let conditions: SQL | undefined = eq(workflow.workspaceId, params.workspaceId)
61+
62+
if (params.level && params.level !== 'all') {
63+
conditions = and(conditions, eq(workflowExecutionLogs.level, params.level))
64+
}
65+
66+
if (params.workflowIds) {
67+
const workflowIds = params.workflowIds.split(',').filter(Boolean)
68+
if (workflowIds.length > 0) conditions = and(conditions, inArray(workflow.id, workflowIds))
69+
}
70+
71+
if (params.folderIds) {
72+
const folderIds = params.folderIds.split(',').filter(Boolean)
73+
if (folderIds.length > 0) conditions = and(conditions, inArray(workflow.folderId, folderIds))
74+
}
75+
76+
if (params.triggers) {
77+
const triggers = params.triggers.split(',').filter(Boolean)
78+
if (triggers.length > 0 && !triggers.includes('all')) {
79+
conditions = and(conditions, inArray(workflowExecutionLogs.trigger, triggers))
80+
}
81+
}
82+
83+
if (params.startDate) {
84+
conditions = and(conditions, gte(workflowExecutionLogs.startedAt, new Date(params.startDate)))
85+
}
86+
if (params.endDate) {
87+
conditions = and(conditions, lte(workflowExecutionLogs.startedAt, new Date(params.endDate)))
88+
}
89+
90+
if (params.search) {
91+
const term = `%${params.search}%`
92+
conditions = and(conditions, sql`${workflowExecutionLogs.executionId} ILIKE ${term}`)
93+
}
94+
if (params.workflowName) {
95+
const nameTerm = `%${params.workflowName}%`
96+
conditions = and(conditions, sql`${workflow.name} ILIKE ${nameTerm}`)
97+
}
98+
if (params.folderName) {
99+
const folderTerm = `%${params.folderName}%`
100+
conditions = and(conditions, sql`${workflow.name} ILIKE ${folderTerm}`)
101+
}
102+
103+
const header = [
104+
'startedAt',
105+
'level',
106+
'workflow',
107+
'trigger',
108+
'durationMs',
109+
'costTotal',
110+
'workflowId',
111+
'executionId',
112+
'message',
113+
'traceSpans',
114+
].join(',')
115+
116+
const encoder = new TextEncoder()
117+
const stream = new ReadableStream<Uint8Array>({
118+
start: async (controller) => {
119+
controller.enqueue(encoder.encode(`${header}\n`))
120+
const pageSize = 1000
121+
let offset = 0
122+
try {
123+
while (true) {
124+
const rows = await db
125+
.select(selectColumns)
126+
.from(workflowExecutionLogs)
127+
.innerJoin(workflow, eq(workflowExecutionLogs.workflowId, workflow.id))
128+
.innerJoin(
129+
permissions,
130+
and(
131+
eq(permissions.entityType, 'workspace'),
132+
eq(permissions.entityId, workflow.workspaceId),
133+
eq(permissions.userId, userId)
134+
)
135+
)
136+
.where(conditions)
137+
.orderBy(desc(workflowExecutionLogs.startedAt))
138+
.limit(pageSize)
139+
.offset(offset)
140+
141+
if (!rows.length) break
142+
143+
for (const r of rows as any[]) {
144+
let message = ''
145+
let traces: any = null
146+
try {
147+
const ed = (r as any).executionData
148+
if (ed) {
149+
if (ed.finalOutput)
150+
message =
151+
typeof ed.finalOutput === 'string'
152+
? ed.finalOutput
153+
: JSON.stringify(ed.finalOutput)
154+
if (ed.message) message = ed.message
155+
if (ed.traceSpans) traces = ed.traceSpans
156+
}
157+
} catch {}
158+
const line = [
159+
escapeCsv(r.startedAt?.toISOString?.() || r.startedAt),
160+
escapeCsv(r.level),
161+
escapeCsv(r.workflowName),
162+
escapeCsv(r.trigger),
163+
escapeCsv(r.totalDurationMs ?? ''),
164+
escapeCsv(r.cost?.total ?? r.cost?.value?.total ?? ''),
165+
escapeCsv(r.workflowId ?? ''),
166+
escapeCsv(r.executionId ?? ''),
167+
escapeCsv(message),
168+
escapeCsv(traces ? JSON.stringify(traces) : ''),
169+
].join(',')
170+
controller.enqueue(encoder.encode(`${line}\n`))
171+
}
172+
173+
offset += pageSize
174+
}
175+
controller.close()
176+
} catch (e: any) {
177+
logger.error('Export stream error', { error: e?.message })
178+
try {
179+
controller.error(e)
180+
} catch {}
181+
}
182+
},
183+
})
184+
185+
const ts = new Date().toISOString().replace(/[:.]/g, '-')
186+
const filename = `logs-${ts}.csv`
187+
188+
return new NextResponse(stream as any, {
189+
status: 200,
190+
headers: {
191+
'Content-Type': 'text/csv; charset=utf-8',
192+
'Content-Disposition': `attachment; filename="${filename}"`,
193+
'Cache-Control': 'no-cache',
194+
},
195+
})
196+
} catch (error: any) {
197+
logger.error('Export error', { error: error?.message })
198+
return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
199+
}
200+
}

‎apps/sim/app/api/logs/route.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ const QueryParamsSchema = z.object({
2222
startDate: z.string().optional(),
2323
endDate: z.string().optional(),
2424
search: z.string().optional(),
25+
workflowName: z.string().optional(),
26+
folderName: z.string().optional(),
2527
workspaceId: z.string(),
2628
})
2729

@@ -155,6 +157,18 @@ export async function GET(request: NextRequest) {
155157
conditions = and(conditions, sql`${workflowExecutionLogs.executionId} ILIKE ${searchTerm}`)
156158
}
157159

160+
// Filter by workflow name (from advanced search input)
161+
if (params.workflowName) {
162+
const nameTerm = `%${params.workflowName}%`
163+
conditions = and(conditions, sql`${workflow.name} ILIKE ${nameTerm}`)
164+
}
165+
166+
// Filter by folder name (best-effort text match when present on workflows)
167+
if (params.folderName) {
168+
const folderTerm = `%${params.folderName}%`
169+
conditions = and(conditions, sql`${workflow.name} ILIKE ${folderTerm}`)
170+
}
171+
158172
// Execute the query using the optimized join
159173
const logs = await baseQuery
160174
.where(conditions)

‎apps/sim/app/workspace/[workspaceId]/logs/components/filters/components/workflow.tsx‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { useEffect, useMemo, useState } from 'react'
22
import { Check, ChevronDown } from 'lucide-react'
3+
import { useParams } from 'next/navigation'
34
import { Button } from '@/components/ui/button'
45
import {
56
Command,
@@ -26,20 +27,27 @@ interface WorkflowOption {
2627
}
2728

2829
export default function Workflow() {
29-
const { workflowIds, toggleWorkflowId, setWorkflowIds } = useFilterStore()
30+
const { workflowIds, toggleWorkflowId, setWorkflowIds, folderIds } = useFilterStore()
31+
const params = useParams()
32+
const workspaceId = params?.workspaceId as string | undefined
3033
const [workflows, setWorkflows] = useState<WorkflowOption[]>([])
3134
const [loading, setLoading] = useState(true)
3235
const [search, setSearch] = useState('')
3336

34-
// Fetch all available workflows from the API
3537
useEffect(() => {
3638
const fetchWorkflows = async () => {
3739
try {
3840
setLoading(true)
39-
const response = await fetch('/api/workflows')
41+
const query = workspaceId ? `?workspaceId=${encodeURIComponent(workspaceId)}` : ''
42+
const response = await fetch(`/api/workflows${query}`)
4043
if (response.ok) {
4144
const { data } = await response.json()
42-
const workflowOptions: WorkflowOption[] = data.map((workflow: any) => ({
45+
const scoped = Array.isArray(data)
46+
? folderIds.length > 0
47+
? data.filter((w: any) => (w.folderId ? folderIds.includes(w.folderId) : false))
48+
: data
49+
: []
50+
const workflowOptions: WorkflowOption[] = scoped.map((workflow: any) => ({
4351
id: workflow.id,
4452
name: workflow.name,
4553
color: workflow.color || '#3972F6',
@@ -54,7 +62,7 @@ export default function Workflow() {
5462
}
5563

5664
fetchWorkflows()
57-
}, [])
65+
}, [workspaceId, folderIds])
5866

5967
const getSelectedWorkflowsText = () => {
6068
if (workflowIds.length === 0) return 'All workflows'

0 commit comments

Comments
 (0)