improvement(async): improve error capturing for asynchronous workflow executions (#1808)

* improvement(async): improve error capturing for asynchronous workflow executions

* surface more erros

* added more logs

* fix failing tests

* ack DB comments

Author: waleedlatif1

apps/sim/app/api/chat/[identifier]/route.ts

@@ -2,7 +2,9 @@ import { db } from '@sim/db'
 import { chat, workflow, workspace } from '@sim/db/schema'
 import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
+import { v4 as uuidv4 } from 'uuid'
 import { createLogger } from '@/lib/logs/console/logger'
+import { LoggingSession } from '@/lib/logs/execution/logging-session'
 import { ChatFiles } from '@/lib/uploads'
 import { generateRequestId } from '@/lib/utils'
 import {
@@ -59,6 +61,29 @@ export async function POST(
 
     if (!deployment.isActive) {
       logger.warn(`[${requestId}] Chat is not active: ${identifier}`)
+
+      const executionId = uuidv4()
+      const loggingSession = new LoggingSession(
+        deployment.workflowId,
+        executionId,
+        'chat',
+        requestId
+      )
+
+      await loggingSession.safeStart({
+        userId: deployment.userId,
+        workspaceId: '', // Will be resolved if needed
+        variables: {},
+      })
+
+      await loggingSession.safeCompleteWithError({
+        error: {
+          message: 'This chat is currently unavailable. The chat has been disabled.',
+          stackTrace: undefined,
+        },
+        traceSpans: [],
+      })
+
       return addCorsHeaders(createErrorResponse('This chat is currently unavailable', 403), request)
     }
 
@@ -96,6 +121,29 @@ export async function POST(
 
     if (workflowResult.length === 0 || !workflowResult[0].isDeployed) {
       logger.warn(`[${requestId}] Workflow not found or not deployed: ${deployment.workflowId}`)
+
+      const executionId = uuidv4()
+      const loggingSession = new LoggingSession(
+        deployment.workflowId,
+        executionId,
+        'chat',
+        requestId
+      )
+
+      await loggingSession.safeStart({
+        userId: deployment.userId,
+        workspaceId: workflowResult[0]?.workspaceId || '',
+        variables: {},
+      })
+
+      await loggingSession.safeCompleteWithError({
+        error: {
+          message: 'Chat workflow is not available. The workflow is not deployed.',
+          stackTrace: undefined,
+        },
+        traceSpans: [],
+      })
+
       return addCorsHeaders(createErrorResponse('Chat workflow is not available', 503), request)
     }
 
@@ -109,6 +157,29 @@ export async function POST(
 
       if (workspaceData.length === 0) {
         logger.error(`[${requestId}] Workspace not found for workflow ${deployment.workflowId}`)
+
+        const executionId = uuidv4()
+        const loggingSession = new LoggingSession(
+          deployment.workflowId,
+          executionId,
+          'chat',
+          requestId
+        )
+
+        await loggingSession.safeStart({
+          userId: deployment.userId,
+          workspaceId: workflowResult[0].workspaceId || '',
+          variables: {},
+        })
+
+        await loggingSession.safeCompleteWithError({
+          error: {
+            message: 'Workspace not found. Critical configuration error - please contact support.',
+            stackTrace: undefined,
+          },
+          traceSpans: [],
+        })
+
         return addCorsHeaders(createErrorResponse('Workspace not found', 500), request)
       }
 
@@ -140,16 +211,43 @@ export async function POST(
           executionId,
         }
 
-        const uploadedFiles = await ChatFiles.processChatFiles(
-          files,
-          executionContext,
-          requestId,
-          deployment.userId
-        )
-
-        if (uploadedFiles.length > 0) {
-          workflowInput.files = uploadedFiles
-          logger.info(`[${requestId}] Successfully processed ${uploadedFiles.length} files`)
+        try {
+          const uploadedFiles = await ChatFiles.processChatFiles(
+            files,
+            executionContext,
+            requestId,
+            deployment.userId
+          )
+
+          if (uploadedFiles.length > 0) {
+            workflowInput.files = uploadedFiles
+            logger.info(`[${requestId}] Successfully processed ${uploadedFiles.length} files`)
+          }
+        } catch (fileError: any) {
+          logger.error(`[${requestId}] Failed to process chat files:`, fileError)
+
+          const fileLoggingSession = new LoggingSession(
+            deployment.workflowId,
+            executionId,
+            'chat',
+            requestId
+          )
+
+          await fileLoggingSession.safeStart({
+            userId: workspaceOwnerId,
+            workspaceId: workflowResult[0].workspaceId || '',
+            variables: {},
+          })
+
+          await fileLoggingSession.safeCompleteWithError({
+            error: {
+              message: `File upload failed: ${fileError.message || 'Unable to process uploaded files'}`,
+              stackTrace: fileError.stack,
+            },
+            traceSpans: [],
+          })
+
+          throw fileError
         }
       }
 

apps/sim/app/api/webhooks/trigger/[path]/route.test.ts

@@ -11,7 +11,13 @@ import {
   mockTriggerDevSdk,
 } from '@/app/api/__test-utils__/utils'
 
-// Prefer mocking the background module to avoid loading Trigger.dev at all during tests
+vi.mock('@trigger.dev/sdk', () => ({
+  tasks: {
+    trigger: vi.fn().mockResolvedValue({ id: 'mock-task-id' }),
+  },
+  task: vi.fn().mockReturnValue({}),
+}))
+
 vi.mock('@/background/webhook-execution', () => ({
   executeWebhookJob: vi.fn().mockResolvedValue({
     success: true,
@@ -22,6 +28,10 @@ vi.mock('@/background/webhook-execution', () => ({
   }),
 }))
 
+vi.mock('@/background/logs-webhook-delivery', () => ({
+  logsWebhookDelivery: {},
+}))
+
 const hasProcessedMessageMock = vi.fn().mockResolvedValue(false)
 const markMessageAsProcessedMock = vi.fn().mockResolvedValue(true)
 const closeRedisConnectionMock = vi.fn().mockResolvedValue(undefined)
@@ -78,27 +88,19 @@ vi.mock('@/executor', () => ({
   })),
 }))
 
-// Set up environment before any imports
 process.env.DATABASE_URL = 'postgresql://test:test@localhost:5432/test'
 
-// Mock postgres dependencies
 vi.mock('drizzle-orm/postgres-js', () => ({
   drizzle: vi.fn().mockReturnValue({}),
 }))
 
 vi.mock('postgres', () => vi.fn().mockReturnValue({}))
 
-// The @sim/db mock is handled in test utils via mockExecutionDependencies()
-
-// (removed duplicate utils mock - defined above with specific handlers)
-
 describe('Webhook Trigger API Route', () => {
   beforeEach(() => {
-    // Ensure a fresh module graph so per-test vi.doMock() takes effect before imports
     vi.resetModules()
     vi.clearAllMocks()
 
-    // Clear global mock data
     globalMockData.webhooks.length = 0
     globalMockData.workflows.length = 0
     globalMockData.schedules.length = 0
@@ -172,43 +174,22 @@ describe('Webhook Trigger API Route', () => {
     vi.clearAllMocks()
   })
 
-  // Removed: WhatsApp verification test has complex mock setup issues
-
-  /**
-   * Test POST webhook with workflow execution
-   * Verifies that a webhook trigger properly initiates workflow execution
-   */
-  // TODO: Fix failing test - returns 500 instead of 200
-  // it('should trigger workflow execution via POST', async () => { ... })
-
-  /**
-   * Test 404 handling for non-existent webhooks
-   */
   it('should handle 404 for non-existent webhooks', async () => {
-    // The global @sim/db mock already returns empty arrays, so findWebhookAndWorkflow will return null
-
-    // Create a mock request
     const req = createMockRequest('POST', { event: 'test' })
 
-    // Mock the path param
     const params = Promise.resolve({ path: 'non-existent-path' })
 
-    // Import the handler
     const { POST } = await import('@/app/api/webhooks/trigger/[path]/route')
 
-    // Call the handler
     const response = await POST(req, { params })
 
-    // Check response - expect 404 since our implementation returns 404 when webhook is not found
     expect(response.status).toBe(404)
 
-    // Parse the response body
     const text = await response.text()
-    expect(text).toMatch(/not found/i) // Response should contain "not found" message
+    expect(text).toMatch(/not found/i)
   })
 
   describe('Generic Webhook Authentication', () => {
-    // Mock billing and rate limiting dependencies
     beforeEach(() => {
       vi.doMock('@/lib/billing/core/subscription', () => ({
         getHighestPrioritySubscription: vi.fn().mockResolvedValue({
@@ -222,11 +203,7 @@ describe('Webhook Trigger API Route', () => {
       }))
     })
 
-    /**
-     * Test generic webhook without authentication (default behavior)
-     */
     it('should process generic webhook without authentication', async () => {
-      // Configure mock data
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
         provider: 'generic',
@@ -249,18 +226,16 @@ describe('Webhook Trigger API Route', () => {
       const { POST } = await import('@/app/api/webhooks/trigger/[path]/route')
       const response = await POST(req, { params })
 
-      // Should succeed (200 OK with webhook processed message)
       expect(response.status).toBe(200)
 
       const data = await response.json()
       expect(data.message).toBe('Webhook processed')
     })
 
     /**
-     * Test generic webhook with Bearer token authentication (no custom header)
+     * Test generic webhook with Bearer token authentication
      */
     it('should authenticate with Bearer token when no custom header is configured', async () => {
-      // Configure mock data with Bearer token
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
         provider: 'generic',
@@ -288,9 +263,6 @@ describe('Webhook Trigger API Route', () => {
       expect(response.status).toBe(200)
     })
 
-    /**
-     * Test generic webhook with custom header authentication
-     */
     it('should authenticate with custom header when configured', async () => {
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
@@ -323,9 +295,6 @@ describe('Webhook Trigger API Route', () => {
       expect(response.status).toBe(200)
     })
 
-    /**
-     * Test case insensitive Bearer token authentication
-     */
     it('should handle case insensitive Bearer token authentication', async () => {
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
@@ -369,9 +338,6 @@ describe('Webhook Trigger API Route', () => {
       }
     })
 
-    /**
-     * Test case insensitive custom header authentication
-     */
     it('should handle case insensitive custom header authentication', async () => {
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
@@ -414,9 +380,6 @@ describe('Webhook Trigger API Route', () => {
       }
     })
 
-    /**
-     * Test rejection of wrong Bearer token
-     */
     it('should reject wrong Bearer token', async () => {
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
@@ -442,9 +405,6 @@ describe('Webhook Trigger API Route', () => {
       expect(processWebhookMock).not.toHaveBeenCalled()
     })
 
-    /**
-     * Test rejection of wrong custom header token
-     */
     it('should reject wrong custom header token', async () => {
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
@@ -474,9 +434,6 @@ describe('Webhook Trigger API Route', () => {
       expect(processWebhookMock).not.toHaveBeenCalled()
     })
 
-    /**
-     * Test rejection of missing authentication
-     */
     it('should reject missing authentication when required', async () => {
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
@@ -498,9 +455,6 @@ describe('Webhook Trigger API Route', () => {
       expect(processWebhookMock).not.toHaveBeenCalled()
     })
 
-    /**
-     * Test exclusivity - Bearer token should be rejected when custom header is configured
-     */
     it('should reject Bearer token when custom header is configured', async () => {
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
@@ -530,9 +484,6 @@ describe('Webhook Trigger API Route', () => {
       expect(processWebhookMock).not.toHaveBeenCalled()
     })
 
-    /**
-     * Test wrong custom header name is rejected
-     */
     it('should reject wrong custom header name', async () => {
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',
@@ -562,9 +513,6 @@ describe('Webhook Trigger API Route', () => {
       expect(processWebhookMock).not.toHaveBeenCalled()
     })
 
-    /**
-     * Test authentication required but no token configured
-     */
     it('should reject when auth is required but no token is configured', async () => {
       globalMockData.webhooks.push({
         id: 'generic-webhook-id',