fix(telemetry): updated telemetry, added nested sanitization, added granular trace spans for logs and updated UI (#1627)

* feat(logs): updated telemetry, added nested sanitization, added granular trace spans for logs and updated UI

* refactor trace spans into separate components

* remove any's from tool defs

* updated UI and overlayed spans

* cleanup

* ack PR comments

* stricter type safety

* clean

Author: waleedlatif1

apps/sim/app/api/knowledge/[id]/documents/route.ts

@@ -186,6 +186,20 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
           `[${requestId}] Starting controlled async processing of ${createdDocuments.length} documents`
         )
 
+        // Track bulk document upload
+        try {
+          const { trackPlatformEvent } = await import('@/lib/telemetry/tracer')
+          trackPlatformEvent('platform.knowledge_base.documents_uploaded', {
+            'knowledge_base.id': knowledgeBaseId,
+            'documents.count': createdDocuments.length,
+            'documents.upload_type': 'bulk',
+            'processing.chunk_size': validatedData.processingOptions.chunkSize,
+            'processing.recipe': validatedData.processingOptions.recipe,
+          })
+        } catch (_e) {
+          // Silently fail
+        }
+
         processDocumentsWithQueue(
           createdDocuments,
           knowledgeBaseId,
@@ -231,6 +245,20 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
 
         const newDocument = await createSingleDocument(validatedData, knowledgeBaseId, requestId)
 
+        // Track single document upload
+        try {
+          const { trackPlatformEvent } = await import('@/lib/telemetry/tracer')
+          trackPlatformEvent('platform.knowledge_base.documents_uploaded', {
+            'knowledge_base.id': knowledgeBaseId,
+            'documents.count': 1,
+            'documents.upload_type': 'single',
+            'document.mime_type': validatedData.mimeType,
+            'document.file_size': validatedData.fileSize,
+          })
+        } catch (_e) {
+          // Silently fail
+        }
+
         return NextResponse.json({
           success: true,
           data: newDocument,

apps/sim/app/api/knowledge/search/route.test.ts

@@ -214,10 +214,6 @@ describe('Knowledge Search API Route', () => {
       const response = await POST(req)
       const data = await response.json()
 
-      if (response.status !== 200) {
-        console.log('Test failed with response:', data)
-      }
-
       expect(response.status).toBe(200)
       expect(data.success).toBe(true)
       expect(data.data.results).toHaveLength(2)
@@ -723,10 +719,6 @@ describe('Knowledge Search API Route', () => {
       const response = await POST(req)
       const data = await response.json()
 
-      if (response.status !== 200) {
-        console.log('Tag-only search test error:', data)
-      }
-
       expect(response.status).toBe(200)
       expect(data.success).toBe(true)
       expect(data.data.results).toHaveLength(2)
@@ -783,10 +775,6 @@ describe('Knowledge Search API Route', () => {
       const response = await POST(req)
       const data = await response.json()
 
-      if (response.status !== 200) {
-        console.log('Query+tag combination test error:', data)
-      }
-
       expect(response.status).toBe(200)
       expect(data.success).toBe(true)
       expect(data.data.results).toHaveLength(2)

apps/sim/app/api/mcp/servers/route.ts

@@ -106,6 +106,20 @@ export const POST = withMcpAuth('write')(
       mcpService.clearCache(workspaceId)
 
       logger.info(`[${requestId}] Successfully registered MCP server: ${body.name}`)
+
+      // Track MCP server registration
+      try {
+        const { trackPlatformEvent } = await import('@/lib/telemetry/tracer')
+        trackPlatformEvent('platform.mcp.server_added', {
+          'mcp.server_id': serverId,
+          'mcp.server_name': body.name,
+          'mcp.transport': body.transport,
+          'workspace.id': workspaceId,
+        })
+      } catch (_e) {
+        // Silently fail
+      }
+
       return createMcpSuccessResponse({ serverId }, 201)
     } catch (error) {
       logger.error(`[${requestId}] Error registering MCP server:`, error)

apps/sim/app/api/mcp/tools/execute/route.ts

@@ -174,6 +174,20 @@ export const POST = withMcpAuth('read')(
         )
       }
       logger.info(`[${requestId}] Successfully executed tool ${toolName} on server ${serverId}`)
+
+      // Track MCP tool execution
+      try {
+        const { trackPlatformEvent } = await import('@/lib/telemetry/tracer')
+        trackPlatformEvent('platform.mcp.tool_executed', {
+          'mcp.server_id': serverId,
+          'mcp.tool_name': toolName,
+          'mcp.execution_status': 'success',
+          'workspace.id': workspaceId,
+        })
+      } catch (_e) {
+        // Silently fail
+      }
+
       return createMcpSuccessResponse(transformedResult)
     } catch (error) {
       logger.error(`[${requestId}] Error executing MCP tool:`, error)

apps/sim/app/api/schedules/route.ts

@@ -366,6 +366,19 @@ export async function POST(req: NextRequest) {
       cronExpression,
     })
 
+    // Track schedule creation/update
+    try {
+      const { trackPlatformEvent } = await import('@/lib/telemetry/tracer')
+      trackPlatformEvent('platform.schedule.created', {
+        'workflow.id': workflowId,
+        'schedule.type': scheduleType || 'daily',
+        'schedule.timezone': timezone,
+        'schedule.is_custom': scheduleType === 'custom',
+      })
+    } catch (_e) {
+      // Silently fail
+    }
+
     return NextResponse.json({
       message: 'Schedule updated',
       nextRunAt,

apps/sim/app/api/telemetry/route.ts

@@ -12,6 +12,7 @@ const ALLOWED_CATEGORIES = [
   'error',
   'workflow',
   'consent',
+  'batch',
 ]
 
 const DEFAULT_TIMEOUT = 5000 // 5 seconds timeout
@@ -132,7 +133,6 @@ async function forwardToCollector(data: any): Promise<boolean> {
       ],
     }
 
-    // Create explicit AbortController for timeout
     const controller = new AbortController()
     const timeoutId = setTimeout(() => controller.abort(), timeout)
 

apps/sim/app/api/templates/[id]/use/route.ts

@@ -174,6 +174,23 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
       `[${requestId}] Successfully used template: ${id}, created workflow: ${newWorkflowId}, database returned: ${result.id}`
     )
 
+    // Track template usage
+    try {
+      const { trackPlatformEvent } = await import('@/lib/telemetry/tracer')
+      const templateState = templateData.state as any
+      trackPlatformEvent('platform.template.used', {
+        'template.id': id,
+        'template.name': templateData.name,
+        'workflow.created_id': newWorkflowId,
+        'workflow.blocks_count': templateState?.blocks
+          ? Object.keys(templateState.blocks).length
+          : 0,
+        'workspace.id': workspaceId,
+      })
+    } catch (_e) {
+      // Silently fail
+    }
+
     // Verify the workflow was actually created
     const verifyWorkflow = await db
       .select({ id: workflow.id })

apps/sim/app/api/workflows/[id]/deploy/route.ts

@@ -353,6 +353,31 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
 
     logger.info(`[${requestId}] Workflow deployed successfully: ${id}`)
 
+    // Track workflow deployment
+    try {
+      const { trackPlatformEvent } = await import('@/lib/telemetry/tracer')
+
+      // Aggregate block types to understand which blocks are being used
+      const blockTypeCounts: Record<string, number> = {}
+      for (const block of Object.values(currentState.blocks)) {
+        const blockType = (block as any).type || 'unknown'
+        blockTypeCounts[blockType] = (blockTypeCounts[blockType] || 0) + 1
+      }
+
+      trackPlatformEvent('platform.workflow.deployed', {
+        'workflow.id': id,
+        'workflow.name': workflowData!.name,
+        'workflow.blocks_count': Object.keys(currentState.blocks).length,
+        'workflow.edges_count': currentState.edges.length,
+        'workflow.has_loops': Object.keys(currentState.loops).length > 0,
+        'workflow.has_parallels': Object.keys(currentState.parallels).length > 0,
+        'workflow.api_key_type': keyInfo?.type || 'default',
+        'workflow.block_types': JSON.stringify(blockTypeCounts),
+      })
+    } catch (_e) {
+      // Silently fail
+    }
+
     const responseApiKeyInfo = keyInfo ? `${keyInfo.name} (${keyInfo.type})` : 'Default key'
 
     return createSuccessResponse({
@@ -400,6 +425,17 @@ export async function DELETE(
     })
 
     logger.info(`[${requestId}] Workflow undeployed successfully: ${id}`)
+
+    // Track workflow undeployment
+    try {
+      const { trackPlatformEvent } = await import('@/lib/telemetry/tracer')
+      trackPlatformEvent('platform.workflow.undeployed', {
+        'workflow.id': id,
+      })
+    } catch (_e) {
+      // Silently fail
+    }
+
     return createSuccessResponse({
       isDeployed: false,
       deployedAt: null,

apps/sim/app/api/workflows/[id]/log/route.ts

@@ -44,20 +44,20 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
         variables: {},
       })
 
-      const { traceSpans } = buildTraceSpans(result)
+      const { traceSpans, totalDuration } = buildTraceSpans(result)
 
       if (result.success === false) {
         const message = result.error || 'Workflow execution failed'
         await loggingSession.safeCompleteWithError({
           endedAt: new Date().toISOString(),
-          totalDurationMs: result.metadata?.duration || 0,
+          totalDurationMs: totalDuration || result.metadata?.duration || 0,
           error: { message },
           traceSpans,
         })
       } else {
         await loggingSession.safeComplete({
           endedAt: new Date().toISOString(),
-          totalDurationMs: result.metadata?.duration || 0,
+          totalDurationMs: totalDuration || result.metadata?.duration || 0,
           finalOutput: result.output || {},
           traceSpans,
         })

apps/sim/app/api/workflows/route.ts

@@ -99,6 +99,19 @@ export async function POST(req: NextRequest) {
 
     logger.info(`[${requestId}] Creating workflow ${workflowId} for user ${session.user.id}`)
 
+    // Track workflow creation
+    try {
+      const { trackPlatformEvent } = await import('@/lib/telemetry/tracer')
+      trackPlatformEvent('platform.workflow.created', {
+        'workflow.id': workflowId,
+        'workflow.name': name,
+        'workflow.has_workspace': !!workspaceId,
+        'workflow.has_folder': !!folderId,
+      })
+    } catch (_e) {
+      // Silently fail
+    }
+
     await db.insert(workflow).values({
       id: workflowId,
       userId: session.user.id,