Feat/copilot files (#886)

* Connects to s3

* Checkpoint

* File shows in message

* Make files clickable

* User input image

* Persist thumbnails

* Drag and drop files

* Lint

* Fix isdev

* Dont re-download files on rerender

Author: Sg312

apps/sim/app/api/copilot/chat/file-utils.ts

@@ -0,0 +1,132 @@
+export interface FileAttachment {
+  id: string
+  s3_key: string
+  filename: string
+  media_type: string
+  size: number
+}
+
+export interface AnthropicMessageContent {
+  type: 'text' | 'image' | 'document'
+  text?: string
+  source?: {
+    type: 'base64'
+    media_type: string
+    data: string
+  }
+}
+
+/**
+ * Mapping of MIME types to Anthropic content types
+ */
+export const MIME_TYPE_MAPPING: Record<string, 'image' | 'document'> = {
+  // Images
+  'image/jpeg': 'image',
+  'image/jpg': 'image',
+  'image/png': 'image',
+  'image/gif': 'image',
+  'image/webp': 'image',
+  'image/svg+xml': 'image',
+
+  // Documents
+  'application/pdf': 'document',
+  'text/plain': 'document',
+  'text/csv': 'document',
+  'application/json': 'document',
+  'application/xml': 'document',
+  'text/xml': 'document',
+  'text/html': 'document',
+  'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'document', // .docx
+  'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'document', // .xlsx
+  'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'document', // .pptx
+  'application/msword': 'document', // .doc
+  'application/vnd.ms-excel': 'document', // .xls
+  'application/vnd.ms-powerpoint': 'document', // .ppt
+  'text/markdown': 'document',
+  'application/rtf': 'document',
+}
+
+/**
+ * Get the Anthropic content type for a given MIME type
+ */
+export function getAnthropicContentType(mimeType: string): 'image' | 'document' | null {
+  return MIME_TYPE_MAPPING[mimeType.toLowerCase()] || null
+}
+
+/**
+ * Check if a MIME type is supported by Anthropic
+ */
+export function isSupportedFileType(mimeType: string): boolean {
+  return mimeType.toLowerCase() in MIME_TYPE_MAPPING
+}
+
+/**
+ * Convert a file buffer to base64
+ */
+export function bufferToBase64(buffer: Buffer): string {
+  return buffer.toString('base64')
+}
+
+/**
+ * Create Anthropic message content from file data
+ */
+export function createAnthropicFileContent(
+  fileBuffer: Buffer,
+  mimeType: string
+): AnthropicMessageContent | null {
+  const contentType = getAnthropicContentType(mimeType)
+  if (!contentType) {
+    return null
+  }
+
+  return {
+    type: contentType,
+    source: {
+      type: 'base64',
+      media_type: mimeType,
+      data: bufferToBase64(fileBuffer),
+    },
+  }
+}
+
+/**
+ * Extract file extension from filename
+ */
+export function getFileExtension(filename: string): string {
+  const lastDot = filename.lastIndexOf('.')
+  return lastDot !== -1 ? filename.slice(lastDot + 1).toLowerCase() : ''
+}
+
+/**
+ * Get MIME type from file extension (fallback if not provided)
+ */
+export function getMimeTypeFromExtension(extension: string): string {
+  const extensionMimeMap: Record<string, string> = {
+    // Images
+    jpg: 'image/jpeg',
+    jpeg: 'image/jpeg',
+    png: 'image/png',
+    gif: 'image/gif',
+    webp: 'image/webp',
+    svg: 'image/svg+xml',
+
+    // Documents
+    pdf: 'application/pdf',
+    txt: 'text/plain',
+    csv: 'text/csv',
+    json: 'application/json',
+    xml: 'application/xml',
+    html: 'text/html',
+    htm: 'text/html',
+    docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+    doc: 'application/msword',
+    xls: 'application/vnd.ms-excel',
+    ppt: 'application/vnd.ms-powerpoint',
+    md: 'text/markdown',
+    rtf: 'application/rtf',
+  }
+
+  return extensionMimeMap[extension.toLowerCase()] || 'application/octet-stream'
+}

apps/sim/app/api/copilot/chat/route.ts

@@ -13,12 +13,25 @@ import { getCopilotModel } from '@/lib/copilot/config'
 import { TITLE_GENERATION_SYSTEM_PROMPT, TITLE_GENERATION_USER_PROMPT } from '@/lib/copilot/prompts'
 import { env } from '@/lib/env'
 import { createLogger } from '@/lib/logs/console/logger'
+import { downloadFile } from '@/lib/uploads'
+import { downloadFromS3WithConfig } from '@/lib/uploads/s3/s3-client'
+import { S3_COPILOT_CONFIG, USE_S3_STORAGE } from '@/lib/uploads/setup'
 import { db } from '@/db'
 import { copilotChats } from '@/db/schema'
 import { executeProviderRequest } from '@/providers'
+import { createAnthropicFileContent, isSupportedFileType } from './file-utils'
 
 const logger = createLogger('CopilotChatAPI')
 
+// Schema for file attachments
+const FileAttachmentSchema = z.object({
+  id: z.string(),
+  s3_key: z.string(),
+  filename: z.string(),
+  media_type: z.string(),
+  size: z.number(),
+})
+
 // Schema for chat messages
 const ChatMessageSchema = z.object({
   message: z.string().min(1, 'Message is required'),
@@ -29,6 +42,7 @@ const ChatMessageSchema = z.object({
   createNewChat: z.boolean().optional().default(false),
   stream: z.boolean().optional().default(true),
   implicitFeedback: z.string().optional(),
+  fileAttachments: z.array(FileAttachmentSchema).optional(),
 })
 
 // Sim Agent API configuration
@@ -145,6 +159,7 @@ export async function POST(req: NextRequest) {
       createNewChat,
       stream,
       implicitFeedback,
+      fileAttachments,
     } = ChatMessageSchema.parse(body)
 
     logger.info(`[${tracker.requestId}] Processing copilot chat request`, {
@@ -195,15 +210,91 @@ export async function POST(req: NextRequest) {
       }
     }
 
+    // Process file attachments if present
+    const processedFileContents: any[] = []
+    if (fileAttachments && fileAttachments.length > 0) {
+      logger.info(`[${tracker.requestId}] Processing ${fileAttachments.length} file attachments`)
+
+      for (const attachment of fileAttachments) {
+        try {
+          // Check if file type is supported
+          if (!isSupportedFileType(attachment.media_type)) {
+            logger.warn(`[${tracker.requestId}] Unsupported file type: ${attachment.media_type}`)
+            continue
+          }
+
+          // Download file from S3
+          logger.info(`[${tracker.requestId}] Downloading file: ${attachment.s3_key}`)
+          let fileBuffer: Buffer
+          if (USE_S3_STORAGE) {
+            fileBuffer = await downloadFromS3WithConfig(attachment.s3_key, S3_COPILOT_CONFIG)
+          } else {
+            // Fallback to generic downloadFile for other storage providers
+            fileBuffer = await downloadFile(attachment.s3_key)
+          }
+
+          // Convert to Anthropic format
+          const fileContent = createAnthropicFileContent(fileBuffer, attachment.media_type)
+          if (fileContent) {
+            processedFileContents.push(fileContent)
+            logger.info(
+              `[${tracker.requestId}] Processed file: ${attachment.filename} (${attachment.media_type})`
+            )
+          }
+        } catch (error) {
+          logger.error(
+            `[${tracker.requestId}] Failed to process file ${attachment.filename}:`,
+            error
+          )
+          // Continue processing other files
+        }
+      }
+    }
+
     // Build messages array for sim agent with conversation history
     const messages = []
 
-    // Add conversation history
+    // Add conversation history (need to rebuild these with file support if they had attachments)
     for (const msg of conversationHistory) {
-      messages.push({
-        role: msg.role,
-        content: msg.content,
-      })
+      if (msg.fileAttachments && msg.fileAttachments.length > 0) {
+        // This is a message with file attachments - rebuild with content array
+        const content: any[] = [{ type: 'text', text: msg.content }]
+
+        // Process file attachments for historical messages
+        for (const attachment of msg.fileAttachments) {
+          try {
+            if (isSupportedFileType(attachment.media_type)) {
+              let fileBuffer: Buffer
+              if (USE_S3_STORAGE) {
+                fileBuffer = await downloadFromS3WithConfig(attachment.s3_key, S3_COPILOT_CONFIG)
+              } else {
+                // Fallback to generic downloadFile for other storage providers
+                fileBuffer = await downloadFile(attachment.s3_key)
+              }
+              const fileContent = createAnthropicFileContent(fileBuffer, attachment.media_type)
+              if (fileContent) {
+                content.push(fileContent)
+              }
+            }
+          } catch (error) {
+            logger.error(
+              `[${tracker.requestId}] Failed to process historical file ${attachment.filename}:`,
+              error
+            )
+          }
+        }
+
+        messages.push({
+          role: msg.role,
+          content,
+        })
+      } else {
+        // Regular text-only message
+        messages.push({
+          role: msg.role,
+          content: msg.content,
+        })
+      }
     }
 
     // Add implicit feedback if provided
@@ -214,11 +305,27 @@ export async function POST(req: NextRequest) {
       })
     }
 
-    // Add current user message
-    messages.push({
-      role: 'user',
-      content: message,
-    })
+    // Add current user message with file attachments
+    if (processedFileContents.length > 0) {
+      // Message with files - use content array format
+      const content: any[] = [{ type: 'text', text: message }]
+
+      // Add file contents
+      for (const fileContent of processedFileContents) {
+        content.push(fileContent)
+      }
+
+      messages.push({
+        role: 'user',
+        content,
+      })
+    } else {
+      // Text-only message
+      messages.push({
+        role: 'user',
+        content: message,
+      })
+    }
 
     // Start title generation in parallel if this is a new chat with first message
     if (actualChatId && !currentChat?.title && conversationHistory.length === 0) {
@@ -270,6 +377,7 @@ export async function POST(req: NextRequest) {
         role: 'user',
         content: message,
         timestamp: new Date().toISOString(),
+        ...(fileAttachments && fileAttachments.length > 0 && { fileAttachments }),
       }
 
       // Create a pass-through stream that captures the response
@@ -590,6 +698,7 @@ export async function POST(req: NextRequest) {
         role: 'user',
         content: message,
         timestamp: new Date().toISOString(),
+        ...(fileAttachments && fileAttachments.length > 0 && { fileAttachments }),
       }
 
       const assistantMessage = {

apps/sim/app/api/copilot/chat/update-messages/route.ts

@@ -24,6 +24,17 @@ const UpdateMessagesSchema = z.object({
       timestamp: z.string(),
       toolCalls: z.array(z.any()).optional(),
       contentBlocks: z.array(z.any()).optional(),
+      fileAttachments: z
+        .array(
+          z.object({
+            id: z.string(),
+            s3_key: z.string(),
+            filename: z.string(),
+            media_type: z.string(),
+            size: z.number(),
+          })
+        )
+        .optional(),
     })
   ),
 })