feat(kb): added support for local file storage for knowledgebase (#1738)

* feat(kb): added support for local file storage for knowledgebase

* updated tests

* ack PR comments

* added back env example

Author: waleedlatif1

apps/sim/.env.example apps/sim/app/.env.exampleapps/sim/.env.example renamed to apps/sim/app/.env.example

@@ -20,5 +20,4 @@ INTERNAL_API_SECRET=your_internal_api_secret # Use `openssl rand -hex 32` to gen
                    # If left commented out, emails will be logged to console instead
 
 # Local AI Models (Optional)
-# OLLAMA_URL=http://localhost:11434  # URL for local Ollama server - uncomment if using local models
-
+# OLLAMA_URL=http://localhost:11434  # URL for local Ollama server - uncomment if using local models

apps/sim/app/api/files/presigned/batch/route.ts

@@ -121,10 +121,24 @@ export async function POST(request: NextRequest) {
     }
 
     if (!isUsingCloudStorage()) {
-      return NextResponse.json(
-        { error: 'Direct uploads are only available when cloud storage is enabled' },
-        { status: 400 }
+      logger.info(
+        `Local storage detected - batch presigned URLs not available, client will use API fallback`
       )
+      return NextResponse.json({
+        files: files.map((file) => ({
+          fileName: file.fileName,
+          presignedUrl: '', // Empty URL signals fallback to API upload
+          fileInfo: {
+            path: '',
+            key: '',
+            name: file.fileName,
+            size: file.fileSize,
+            type: file.contentType,
+          },
+          directUploadSupported: false,
+        })),
+        directUploadSupported: false,
+      })
     }
 
     const storageProvider = getStorageProvider()

apps/sim/app/api/files/presigned/route.test.ts

@@ -25,7 +25,7 @@ describe('/api/files/presigned', () => {
   })
 
   describe('POST', () => {
-    it('should return error when cloud storage is not enabled', async () => {
+    it('should return graceful fallback response when cloud storage is not enabled', async () => {
       setupFileApiMocks({
         cloudEnabled: false,
         storageProvider: 's3',
@@ -45,10 +45,14 @@ describe('/api/files/presigned', () => {
       const response = await POST(request)
       const data = await response.json()
 
-      expect(response.status).toBe(500)
-      expect(data.error).toBe('Direct uploads are only available when cloud storage is enabled')
-      expect(data.code).toBe('STORAGE_CONFIG_ERROR')
+      expect(response.status).toBe(200)
       expect(data.directUploadSupported).toBe(false)
+      expect(data.presignedUrl).toBe('')
+      expect(data.fileName).toBe('test.txt')
+      expect(data.fileInfo).toBeDefined()
+      expect(data.fileInfo.name).toBe('test.txt')
+      expect(data.fileInfo.size).toBe(1024)
+      expect(data.fileInfo.type).toBe('text/plain')
     })
 
     it('should return error when fileName is missing', async () => {

apps/sim/app/api/files/presigned/route.ts

@@ -141,9 +141,21 @@ export async function POST(request: NextRequest) {
     }
 
     if (!isUsingCloudStorage()) {
-      throw new StorageConfigError(
-        'Direct uploads are only available when cloud storage is enabled'
+      logger.info(
+        `Local storage detected - presigned URL not available for ${fileName}, client will use API fallback`
       )
+      return NextResponse.json({
+        fileName,
+        presignedUrl: '', // Empty URL signals fallback to API upload
+        fileInfo: {
+          path: '',
+          key: '',
+          name: fileName,
+          size: fileSize,
+          type: contentType,
+        },
+        directUploadSupported: false,
+      })
     }
 
     const storageProvider = getStorageProvider()

apps/sim/lib/uploads/storage-client.ts

@@ -92,9 +92,34 @@ export async function uploadFile(
     return uploadToS3(file, fileName, contentType, configOrSize)
   }
 
-  throw new Error(
-    'No storage provider configured. Set Azure credentials (AZURE_CONNECTION_STRING or AZURE_ACCOUNT_NAME + AZURE_ACCOUNT_KEY) or configure AWS credentials for S3.'
-  )
+  logger.info(`Uploading file to local storage: ${fileName}`)
+  const { writeFile } = await import('fs/promises')
+  const { join } = await import('path')
+  const { v4: uuidv4 } = await import('uuid')
+  const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/setup.server')
+
+  const safeFileName = fileName.replace(/[^a-zA-Z0-9.-]/g, '_').replace(/\.\./g, '')
+  const uniqueKey = `${uuidv4()}-${safeFileName}`
+  const filePath = join(UPLOAD_DIR_SERVER, uniqueKey)
+
+  try {
+    await writeFile(filePath, file)
+  } catch (error) {
+    logger.error(`Failed to write file to local storage: ${fileName}`, error)
+    throw new Error(
+      `Failed to write file to local storage: ${error instanceof Error ? error.message : 'Unknown error'}`
+    )
+  }
+
+  const fileSize = typeof configOrSize === 'number' ? configOrSize : size || file.length
+
+  return {
+    path: `/api/files/serve/${uniqueKey}`,
+    key: uniqueKey,
+    name: fileName,
+    size: fileSize,
+    type: contentType,
+  }
 }
 
 /**
@@ -144,9 +169,28 @@ export async function downloadFile(
     return downloadFromS3(key)
   }
 
-  throw new Error(
-    'No storage provider configured. Set Azure credentials (AZURE_CONNECTION_STRING or AZURE_ACCOUNT_NAME + AZURE_ACCOUNT_KEY) or configure AWS credentials for S3.'
-  )
+  logger.info(`Downloading file from local storage: ${key}`)
+  const { readFile } = await import('fs/promises')
+  const { join, resolve, sep } = await import('path')
+  const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/setup.server')
+
+  const safeKey = key.replace(/\.\./g, '').replace(/[/\\]/g, '')
+  const filePath = join(UPLOAD_DIR_SERVER, safeKey)
+
+  const resolvedPath = resolve(filePath)
+  const allowedDir = resolve(UPLOAD_DIR_SERVER)
+  if (!resolvedPath.startsWith(allowedDir + sep) && resolvedPath !== allowedDir) {
+    throw new Error('Invalid file path')
+  }
+
+  try {
+    return await readFile(filePath)
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      throw new Error(`File not found: ${key}`)
+    }
+    throw error
+  }
 }
 
 /**
@@ -166,9 +210,29 @@ export async function deleteFile(key: string): Promise<void> {
     return deleteFromS3(key)
   }
 
-  throw new Error(
-    'No storage provider configured. Set Azure credentials (AZURE_CONNECTION_STRING or AZURE_ACCOUNT_NAME + AZURE_ACCOUNT_KEY) or configure AWS credentials for S3.'
-  )
+  logger.info(`Deleting file from local storage: ${key}`)
+  const { unlink } = await import('fs/promises')
+  const { join, resolve, sep } = await import('path')
+  const { UPLOAD_DIR_SERVER } = await import('@/lib/uploads/setup.server')
+
+  const safeKey = key.replace(/\.\./g, '').replace(/[/\\]/g, '')
+  const filePath = join(UPLOAD_DIR_SERVER, safeKey)
+
+  const resolvedPath = resolve(filePath)
+  const allowedDir = resolve(UPLOAD_DIR_SERVER)
+  if (!resolvedPath.startsWith(allowedDir + sep) && resolvedPath !== allowedDir) {
+    throw new Error('Invalid file path')
+  }
+
+  try {
+    await unlink(filePath)
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      logger.warn(`File not found during deletion: ${key}`)
+      return
+    }
+    throw error
+  }
 }
 
 /**
@@ -190,9 +254,8 @@ export async function getPresignedUrl(key: string, expiresIn = 3600): Promise<st
     return getS3PresignedUrl(key, expiresIn)
   }
 
-  throw new Error(
-    'No storage provider configured. Set Azure credentials (AZURE_CONNECTION_STRING or AZURE_ACCOUNT_NAME + AZURE_ACCOUNT_KEY) or configure AWS credentials for S3.'
-  )
+  logger.info(`Generating serve path for local storage: ${key}`)
+  return `/api/files/serve/${encodeURIComponent(key)}`
 }
 
 /**