feat(execution-filesystem): system to pass files between blocks (#866)

* feat(files): pass files between blocks

* presigned URL for downloads

* Remove latest migration before merge

* starter block file upload wasn't getting logged

* checkpoint in human readable form

* checkpoint files / file type outputs

* file downloads working for block outputs

* checkpoint file download

* fix type issues

* remove filereference interface with simpler user file interface

* show files in the tag dropdown for start block

* more migration to simple url object, reduce presigned time to 5 min

* Remove migration 0065_parallel_nightmare and related files

- Deleted apps/sim/db/migrations/0065_parallel_nightmare.sql
- Deleted apps/sim/db/migrations/meta/0065_snapshot.json
- Removed 0065 entry from apps/sim/db/migrations/meta/_journal.json

Preparing for merge with origin/staging and migration regeneration

* add migration files

* fix tests

* Update apps/sim/lib/uploads/setup.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* Update apps/sim/lib/workflows/execution-file-storage.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* Update apps/sim/lib/workflows/execution-file-storage.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* cleanup types

* fix lint

* fix logs typing for file refs

* open download in new tab

* fixed

* Update apps/sim/tools/index.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* fix file block

* cleanup unused code

* fix bugs

* remove hacky file id logic

* fix drag and drop

* fix tests

---------

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

Author: icecrasher321

apps/docs/content/docs/triggers/starter.mdx

@@ -50,7 +50,7 @@ Choose your input method from the dropdown:
         <video autoPlay loop muted playsInline className="w-full -mb-2 rounded-lg" src="/chat-input.mp4"></video>
       </div>
 
-      <p className="text-sm text-gray-600">Chat with your workflow and access both input text and conversation ID for context-aware responses.</p>
+      <p className="text-sm text-gray-600">Chat with your workflow and access input text, conversation ID, and uploaded files for context-aware responses.</p>
     </div>
   </Tab>
 </Tabs>
@@ -60,13 +60,15 @@ Choose your input method from the dropdown:
 In Chat mode, access user input and conversation context through special variables:
 
 ```yaml
-# Reference the chat input and conversation ID in your workflow
+# Reference the chat input, conversation ID, and files in your workflow
 user_message: "<start.input>"
 conversation_id: "<start.conversationId>"
+uploaded_files: "<start.files>"
 ```
 
 - **`<start.input>`** - Contains the user's message text
 - **`<start.conversationId>`** - Unique identifier for the conversation thread
+- **`<start.files>`** - Array of files uploaded by the user (if any)
 
 ## API Execution
 

apps/sim/app/api/__test-utils__/utils.ts

@@ -712,6 +712,7 @@ export function mockFileSystem(
       }
       return Promise.reject(new Error('File not found'))
     }),
+    mkdir: vi.fn().mockResolvedValue(undefined),
   }))
 }
 
@@ -761,14 +762,15 @@ export function createStorageProviderMocks(options: StorageProviderMockOptions =
     getStorageProvider: vi.fn().mockReturnValue(provider),
     isUsingCloudStorage: vi.fn().mockReturnValue(isCloudEnabled),
     uploadFile: vi.fn().mockResolvedValue({
-      path: '/api/files/serve/test-key',
-      key: 'test-key',
+      path: '/api/files/serve/test-key.txt',
+      key: 'test-key.txt',
       name: 'test.txt',
       size: 100,
       type: 'text/plain',
     }),
     downloadFile: vi.fn().mockResolvedValue(Buffer.from('test content')),
     deleteFile: vi.fn().mockResolvedValue(undefined),
+    getPresignedUrl: vi.fn().mockResolvedValue(presignedUrl),
   }))
 
   if (provider === 's3') {
@@ -1235,14 +1237,15 @@ export function setupFileApiMocks(
       getStorageProvider: vi.fn().mockReturnValue('local'),
       isUsingCloudStorage: vi.fn().mockReturnValue(cloudEnabled),
       uploadFile: vi.fn().mockResolvedValue({
-        path: '/api/files/serve/test-key',
-        key: 'test-key',
+        path: '/api/files/serve/test-key.txt',
+        key: 'test-key.txt',
         name: 'test.txt',
         size: 100,
         type: 'text/plain',
       }),
       downloadFile: vi.fn().mockResolvedValue(Buffer.from('test content')),
       deleteFile: vi.fn().mockResolvedValue(undefined),
+      getPresignedUrl: vi.fn().mockResolvedValue('https://example.com/presigned-url'),
     }))
   }
 
@@ -1347,8 +1350,8 @@ export function mockUploadUtils(
   const {
     isCloudStorage = false,
     uploadResult = {
-      path: '/api/files/serve/test-key',
-      key: 'test-key',
+      path: '/api/files/serve/test-key.txt',
+      key: 'test-key.txt',
       name: 'test.txt',
       size: 100,
       type: 'text/plain',

apps/sim/app/api/files/download/route.ts

@@ -0,0 +1,99 @@
+import { type NextRequest, NextResponse } from 'next/server'
+import { createLogger } from '@/lib/logs/console/logger'
+import { getPresignedUrl, getPresignedUrlWithConfig, isUsingCloudStorage } from '@/lib/uploads'
+import { BLOB_EXECUTION_FILES_CONFIG, S3_EXECUTION_FILES_CONFIG } from '@/lib/uploads/setup'
+import { createErrorResponse } from '@/app/api/files/utils'
+
+const logger = createLogger('FileDownload')
+
+export const dynamic = 'force-dynamic'
+
+export async function POST(request: NextRequest) {
+  try {
+    const body = await request.json()
+    const { key, name, storageProvider, bucketName, isExecutionFile } = body
+
+    if (!key) {
+      return createErrorResponse(new Error('File key is required'), 400)
+    }
+
+    logger.info(`Generating download URL for file: ${name || key}`)
+
+    if (isUsingCloudStorage()) {
+      // Generate a fresh 5-minute presigned URL for cloud storage
+      try {
+        let downloadUrl: string
+
+        // Use execution files storage if flagged as execution file
+        if (isExecutionFile) {
+          logger.info(`Using execution files storage for file: ${key}`)
+          downloadUrl = await getPresignedUrlWithConfig(
+            key,
+            {
+              bucket: S3_EXECUTION_FILES_CONFIG.bucket,
+              region: S3_EXECUTION_FILES_CONFIG.region,
+            },
+            5 * 60 // 5 minutes
+          )
+        } else if (storageProvider && (storageProvider === 's3' || storageProvider === 'blob')) {
+          // Use explicitly specified storage provider (legacy support)
+          logger.info(`Using specified storage provider '${storageProvider}' for file: ${key}`)
+
+          if (storageProvider === 's3') {
+            downloadUrl = await getPresignedUrlWithConfig(
+              key,
+              {
+                bucket: bucketName || S3_EXECUTION_FILES_CONFIG.bucket,
+                region: S3_EXECUTION_FILES_CONFIG.region,
+              },
+              5 * 60 // 5 minutes
+            )
+          } else {
+            // blob
+            downloadUrl = await getPresignedUrlWithConfig(
+              key,
+              {
+                accountName: BLOB_EXECUTION_FILES_CONFIG.accountName,
+                accountKey: BLOB_EXECUTION_FILES_CONFIG.accountKey,
+                connectionString: BLOB_EXECUTION_FILES_CONFIG.connectionString,
+                containerName: bucketName || BLOB_EXECUTION_FILES_CONFIG.containerName,
+              },
+              5 * 60 // 5 minutes
+            )
+          }
+        } else {
+          // Use default storage (regular uploads)
+          logger.info(`Using default storage for file: ${key}`)
+          downloadUrl = await getPresignedUrl(key, 5 * 60) // 5 minutes
+        }
+
+        return NextResponse.json({
+          downloadUrl,
+          expiresIn: 300, // 5 minutes in seconds
+          fileName: name || key.split('/').pop() || 'download',
+        })
+      } catch (error) {
+        logger.error(`Failed to generate presigned URL for ${key}:`, error)
+        return createErrorResponse(
+          error instanceof Error ? error : new Error('Failed to generate download URL'),
+          500
+        )
+      }
+    } else {
+      // For local storage, return the direct path
+      const downloadUrl = `${process.env.NEXT_PUBLIC_APP_URL || 'http://localhost:3000'}/api/files/serve/${key}`
+
+      return NextResponse.json({
+        downloadUrl,
+        expiresIn: null, // Local URLs don't expire
+        fileName: name || key.split('/').pop() || 'download',
+      })
+    }
+  } catch (error) {
+    logger.error('Error in file download endpoint:', error)
+    return createErrorResponse(
+      error instanceof Error ? error : new Error('Internal server error'),
+      500
+    )
+  }
+}

apps/sim/app/api/files/execution/[executionId]/[fileId]/route.ts

@@ -0,0 +1,70 @@
+import { type NextRequest, NextResponse } from 'next/server'
+import { createLogger } from '@/lib/logs/console/logger'
+import { generateExecutionFileDownloadUrl } from '@/lib/workflows/execution-file-storage'
+import { getExecutionFiles } from '@/lib/workflows/execution-files-server'
+import type { UserFile } from '@/executor/types'
+
+const logger = createLogger('ExecutionFileDownloadAPI')
+
+/**
+ * Generate a short-lived presigned URL for secure execution file download
+ * GET /api/files/execution/[executionId]/[fileId]
+ */
+export async function GET(
+  request: NextRequest,
+  { params }: { params: Promise<{ executionId: string; fileId: string }> }
+) {
+  try {
+    const { executionId, fileId } = await params
+
+    if (!executionId || !fileId) {
+      return NextResponse.json({ error: 'Execution ID and File ID are required' }, { status: 400 })
+    }
+
+    logger.info(`Generating download URL for file ${fileId} in execution ${executionId}`)
+
+    // Get files for this execution
+    const executionFiles = await getExecutionFiles(executionId)
+
+    if (executionFiles.length === 0) {
+      return NextResponse.json({ error: 'No files found for this execution' }, { status: 404 })
+    }
+
+    // Find the specific file
+    const file = executionFiles.find((f) => f.id === fileId)
+    if (!file) {
+      return NextResponse.json({ error: 'File not found in this execution' }, { status: 404 })
+    }
+
+    // Check if file is expired
+    if (new Date(file.expiresAt) < new Date()) {
+      return NextResponse.json({ error: 'File has expired' }, { status: 410 })
+    }
+
+    // Since ExecutionFileMetadata is now just UserFile, no conversion needed
+    const userFile: UserFile = file
+
+    // Generate a new short-lived presigned URL (5 minutes)
+    const downloadUrl = await generateExecutionFileDownloadUrl(userFile)
+
+    logger.info(`Generated download URL for file ${file.name} (execution: ${executionId})`)
+
+    const response = NextResponse.json({
+      downloadUrl,
+      fileName: file.name,
+      fileSize: file.size,
+      fileType: file.type,
+      expiresIn: 300, // 5 minutes
+    })
+
+    // Ensure no caching of download URLs
+    response.headers.set('Cache-Control', 'no-cache, no-store, must-revalidate')
+    response.headers.set('Pragma', 'no-cache')
+    response.headers.set('Expires', '0')
+
+    return response
+  } catch (error) {
+    logger.error('Error generating execution file download URL:', error)
+    return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
+  }
+}

apps/sim/app/api/files/parse/route.ts

@@ -7,7 +7,7 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { isSupportedFileType, parseFile } from '@/lib/file-parsers'
 import { createLogger } from '@/lib/logs/console/logger'
 import { downloadFile, isUsingCloudStorage } from '@/lib/uploads'
-import { UPLOAD_DIR } from '@/lib/uploads/setup'
+import { UPLOAD_DIR_SERVER } from '@/lib/uploads/setup.server'
 import '@/lib/uploads/setup.server'
 
 export const dynamic = 'force-dynamic'
@@ -70,7 +70,7 @@ export async function POST(request: NextRequest) {
     const requestData = await request.json()
     const { filePath, fileType } = requestData
 
-    if (!filePath) {
+    if (!filePath || (typeof filePath === 'string' && filePath.trim() === '')) {
       return NextResponse.json({ success: false, error: 'No file path provided' }, { status: 400 })
     }
 
@@ -80,6 +80,16 @@ export async function POST(request: NextRequest) {
     if (Array.isArray(filePath)) {
       const results = []
       for (const path of filePath) {
+        // Skip empty or invalid paths
+        if (!path || (typeof path === 'string' && path.trim() === '')) {
+          results.push({
+            success: false,
+            error: 'Empty file path in array',
+            filePath: path || '',
+          })
+          continue
+        }
+
         const result = await parseFileSingle(path, fileType)
         // Add processing time to metadata
         if (result.metadata) {
@@ -154,6 +164,15 @@ export async function POST(request: NextRequest) {
 async function parseFileSingle(filePath: string, fileType?: string): Promise<ParseResult> {
   logger.info('Parsing file:', filePath)
 
+  // Validate that filePath is not empty
+  if (!filePath || filePath.trim() === '') {
+    return {
+      success: false,
+      error: 'Empty file path provided',
+      filePath: filePath || '',
+    }
+  }
+
   // Validate path for security before any processing
   const pathValidation = validateFilePath(filePath)
   if (!pathValidation.isValid) {
@@ -337,7 +356,7 @@ async function handleLocalFile(filePath: string, fileType?: string): Promise<Par
   try {
     // Extract filename from path
     const filename = filePath.split('/').pop() || filePath
-    const fullPath = path.join(UPLOAD_DIR, filename)
+    const fullPath = path.join(UPLOAD_DIR_SERVER, filename)
 
     logger.info('Processing local file:', fullPath)
 

apps/sim/app/api/files/presigned/route.ts

@@ -4,8 +4,7 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { v4 as uuidv4 } from 'uuid'
 import { createLogger } from '@/lib/logs/console/logger'
 import { getStorageProvider, isUsingCloudStorage } from '@/lib/uploads'
-import { getBlobServiceClient } from '@/lib/uploads/blob/blob-client'
-import { getS3Client, sanitizeFilenameForMetadata } from '@/lib/uploads/s3/s3-client'
+// Dynamic imports for storage clients to avoid client-side bundling
 import {
   BLOB_CHAT_CONFIG,
   BLOB_CONFIG,
@@ -169,6 +168,7 @@ async function handleS3PresignedUrl(
 
     const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}`
 
+    const { sanitizeFilenameForMetadata } = await import('@/lib/uploads/s3/s3-client')
     const sanitizedOriginalName = sanitizeFilenameForMetadata(fileName)
 
     const metadata: Record<string, string> = {
@@ -194,6 +194,7 @@ async function handleS3PresignedUrl(
 
     let presignedUrl: string
     try {
+      const { getS3Client } = await import('@/lib/uploads/s3/s3-client')
       presignedUrl = await getSignedUrl(getS3Client(), command, { expiresIn: 3600 })
     } catch (s3Error) {
       logger.error('Failed to generate S3 presigned URL:', s3Error)
@@ -272,6 +273,7 @@ async function handleBlobPresignedUrl(
 
     const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}`
 
+    const { getBlobServiceClient } = await import('@/lib/uploads/blob/blob-client')
     const blobServiceClient = getBlobServiceClient()
     const containerClient = blobServiceClient.getContainerClient(config.containerName)
     const blockBlobClient = containerClient.getBlockBlobClient(uniqueKey)

apps/sim/app/api/files/upload/route.test.ts

@@ -26,7 +26,9 @@ describe('File Upload API Route', () => {
 
   beforeEach(() => {
     vi.resetModules()
-    vi.doMock('@/lib/uploads/setup.server', () => ({}))
+    vi.doMock('@/lib/uploads/setup.server', () => ({
+      UPLOAD_DIR_SERVER: '/tmp/test-uploads',
+    }))
   })
 
   afterEach(() => {
@@ -52,15 +54,22 @@ describe('File Upload API Route', () => {
     const response = await POST(req)
     const data = await response.json()
 
+    // Log error details if test fails
+    if (response.status !== 200) {
+      console.error('Upload failed with status:', response.status)
+      console.error('Error response:', data)
+    }
+
     expect(response.status).toBe(200)
     expect(data).toHaveProperty('path')
     expect(data.path).toMatch(/\/api\/files\/serve\/.*\.txt$/)
     expect(data).toHaveProperty('name', 'test.txt')
     expect(data).toHaveProperty('size')
     expect(data).toHaveProperty('type', 'text/plain')
 
-    const fs = await import('fs/promises')
-    expect(fs.writeFile).toHaveBeenCalled()
+    // Verify the upload function was called (we're mocking at the uploadFile level)
+    const { uploadFile } = await import('@/lib/uploads')
+    expect(uploadFile).toHaveBeenCalled()
   })
 
   it('should upload a file to S3 when in S3 mode', async () => {