fix(copilot): deprecate yaml, json import/export, deprecate build_workflow tool, convert copilot to json-based (#1488)

* Temp commit

* Edit workflow self contained

* Remove build_workflow

* Base bad version

* fix lint

* Sanitize workflows for copilot

* Fixes

* Fix import/export buttons

* fix autolayout

* fix lint

* fix training logic to work with json

* Add claude sonnet 4.5 to copilot

* Lint

* Update copilot url

* Update default model and fix build errors

* Fix tests

---------

Co-authored-by: Vikhyath Mondreti <[email protected]>

Author: Sg312

apps/docs/content/docs/en/yaml/blocks/loop.mdx

@@ -10,7 +10,6 @@ type: object
 required:
   - type
   - name
-  - inputs
   - connections
 properties:
   type:
@@ -22,21 +21,23 @@ properties:
     description: Display name for this loop block
   inputs:
     type: object
-    required:
-      - loopType
+    description: Optional. If omitted, defaults will be applied.
     properties:
       loopType:
         type: string
         enum: [for, forEach]
         description: Type of loop to execute
+        default: for
       iterations:
         type: number
         description: Number of iterations (for 'for' loops)
+        default: 5
         minimum: 1
         maximum: 1000
       collection:
         type: string
         description: Collection to iterate over (for 'forEach' loops)
+        default: ""
       maxConcurrency:
         type: number
         description: Maximum concurrent executions
@@ -45,40 +46,48 @@ properties:
         maximum: 10
   connections:
     type: object
-    required:
-      - loop
     properties:
+      # Nested format (recommended)
       loop:
         type: object
-        required:
-          - start
         properties:
           start:
             type: string
             description: Target block ID to execute inside the loop
           end:
             type: string
             description: Target block ID for loop completion (optional)
+      # Direct handle format (alternative)
+      loop-start-source:
+        type: string | string[]
+        description: Target block ID to execute inside the loop (direct format)
+      loop-end-source:
+        type: string | string[]
+        description: Target block ID for loop completion (direct format, optional)
       error:
         type: string
         description: Target block ID for error handling
+    note: Use either the nested 'loop' format OR the direct 'loop-start-source' format, not both
 ```
 
 ## Connection Configuration
 
-Loop blocks use a special connection format with a `loop` section:
+Loop blocks support two connection formats:
+
+### Direct Handle Format (Alternative)
 
 ```yaml
 connections:
-  loop:
-    start: <string>                     # Target block ID to execute inside the loop
-    end: <string>                       # Target block ID after loop completion (optional)
+  loop-start-source: <string>           # Target block ID to execute inside the loop
+  loop-end-source: <string>             # Target block ID after loop completion (optional)
   error: <string>                       # Target block ID for error handling (optional)
 ```
 
+Both formats work identically. Use whichever you prefer.
+
 ## Child Block Configuration
 
-Blocks inside a loop must have their `parentId` set to the loop block ID:
+Blocks inside a loop must have their `parentId` set to the loop block ID. The `extent` property is automatically set to `'parent'` and doesn't need to be specified:
 
 ```yaml
 loop-1:
@@ -261,6 +270,59 @@ process-task:
     success: task-completed
 ```
 
+### Direct Handle Format Example
+
+The same loop can be written using the direct handle format:
+
+```yaml
+my-loop:
+  type: loop
+  name: "Process Items"
+  inputs:
+    loopType: forEach
+    collection: <start.items>
+  connections:
+    loop-start-source: process-item      # Direct handle format
+    loop-end-source: final-results       # Direct handle format
+    error: handle-error
+
+process-item:
+  type: agent
+  name: "Process Item"
+  parentId: my-loop
+  inputs:
+    systemPrompt: "Process this item"
+    userPrompt: <loop.currentItem>
+    model: gpt-4o
+    apiKey: '{{OPENAI_API_KEY}}'
+```
+
+### Minimal Loop Example (Using Defaults)
+
+You can omit the `inputs` section entirely, and defaults will be applied:
+
+```yaml
+simple-loop:
+  type: loop
+  name: "Simple Loop"
+  # No inputs section - defaults to loopType: 'for', iterations: 5
+  connections:
+    loop-start-source: process-step
+    loop-end-source: complete
+
+process-step:
+  type: agent
+  name: "Process Step"
+  parentId: simple-loop
+  inputs:
+    systemPrompt: "Execute step"
+    userPrompt: "Step <loop.index>"
+    model: gpt-4o
+    apiKey: '{{OPENAI_API_KEY}}'
+```
+
+This loop will execute 5 iterations by default.
+
 ## Loop Variables
 
 Inside loop child blocks, these special variables are available:

apps/sim/app/api/copilot/chat/route.test.ts

@@ -230,7 +230,7 @@ describe('Copilot Chat API Route', () => {
             userId: 'user-123',
             stream: true,
             streamToolCalls: true,
-            model: 'gpt-5',
+            model: 'claude-4.5-sonnet',
             mode: 'agent',
             messageId: 'mock-uuid-1234-5678',
             version: '1.0.0',
@@ -300,7 +300,7 @@ describe('Copilot Chat API Route', () => {
             userId: 'user-123',
             stream: true,
             streamToolCalls: true,
-            model: 'gpt-5',
+            model: 'claude-4.5-sonnet',
             mode: 'agent',
             messageId: 'mock-uuid-1234-5678',
             version: '1.0.0',
@@ -358,7 +358,7 @@ describe('Copilot Chat API Route', () => {
             userId: 'user-123',
             stream: true,
             streamToolCalls: true,
-            model: 'gpt-5',
+            model: 'claude-4.5-sonnet',
             mode: 'agent',
             messageId: 'mock-uuid-1234-5678',
             version: '1.0.0',
@@ -450,7 +450,7 @@ describe('Copilot Chat API Route', () => {
             userId: 'user-123',
             stream: true,
             streamToolCalls: true,
-            model: 'gpt-5',
+            model: 'claude-4.5-sonnet',
             mode: 'ask',
             messageId: 'mock-uuid-1234-5678',
             version: '1.0.0',

apps/sim/app/api/copilot/chat/route.ts

@@ -48,10 +48,11 @@ const ChatMessageSchema = z.object({
       'gpt-4.1',
       'o3',
       'claude-4-sonnet',
+      'claude-4.5-sonnet',
       'claude-4.1-opus',
     ])
     .optional()
-    .default('gpt-5'),
+    .default('claude-4.5-sonnet'),
   mode: z.enum(['ask', 'agent']).optional().default('agent'),
   prefetch: z.boolean().optional(),
   createNewChat: z.boolean().optional().default(false),

apps/sim/app/api/copilot/training/route.ts

@@ -51,13 +51,9 @@ export async function POST(request: NextRequest) {
 
     logger.info('Sending training data to agent indexer', {
       title,
-      operationsCount: operations.length,
+      operationsCount: Array.isArray(operations) ? operations.length : 0,
     })
 
-    const wrappedOperations = {
-      operations: operations,
-    }
-
     // Forward to agent indexer
     const upstreamUrl = `${baseUrl}/operations/add`
     const upstreamResponse = await fetch(upstreamUrl, {
@@ -71,7 +67,7 @@ export async function POST(request: NextRequest) {
         prompt,
         input,
         output,
-        operations: wrappedOperations,
+        operations: { operations },
       }),
     })
 

apps/sim/app/api/workflows/[id]/autolayout/route.ts

@@ -6,13 +6,9 @@ import { z } from 'zod'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
 import { getUserEntityPermissions } from '@/lib/permissions/utils'
-import { simAgentClient } from '@/lib/sim-agent/client'
 import { generateRequestId } from '@/lib/utils'
+import { applyAutoLayout } from '@/lib/workflows/autolayout'
 import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/db-helpers'
-import { getAllBlocks } from '@/blocks/registry'
-import type { BlockConfig } from '@/blocks/types'
-import { resolveOutputType } from '@/blocks/utils'
-import { generateLoopBlocks, generateParallelBlocks } from '@/stores/workflows/workflow/utils'
 
 export const dynamic = 'force-dynamic'
 
@@ -120,123 +116,46 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
       return NextResponse.json({ error: 'Could not load workflow data' }, { status: 500 })
     }
 
-    // Create workflow state for autolayout
-    const workflowState = {
-      blocks: currentWorkflowData.blocks,
-      edges: currentWorkflowData.edges,
-      loops: currentWorkflowData.loops || {},
-      parallels: currentWorkflowData.parallels || {},
-    }
-
     const autoLayoutOptions = {
-      strategy: layoutOptions.strategy,
-      direction: layoutOptions.direction,
-      spacing: {
-        horizontal: layoutOptions.spacing?.horizontal || 500,
-        vertical: layoutOptions.spacing?.vertical || 400,
-        layer: layoutOptions.spacing?.layer || 700,
-      },
-      alignment: layoutOptions.alignment,
+      horizontalSpacing: layoutOptions.spacing?.horizontal || 550,
+      verticalSpacing: layoutOptions.spacing?.vertical || 200,
       padding: {
-        x: layoutOptions.padding?.x || 250,
-        y: layoutOptions.padding?.y || 250,
+        x: layoutOptions.padding?.x || 150,
+        y: layoutOptions.padding?.y || 150,
       },
+      alignment: layoutOptions.alignment,
     }
 
-    // Gather block registry and utilities for sim-agent
-    const blocks = getAllBlocks()
-    const blockRegistry = blocks.reduce(
-      (acc, block) => {
-        const blockType = block.type
-        acc[blockType] = {
-          ...block,
-          id: blockType,
-          subBlocks: block.subBlocks || [],
-          outputs: block.outputs || {},
-        } as any
-        return acc
-      },
-      {} as Record<string, BlockConfig>
+    const layoutResult = applyAutoLayout(
+      currentWorkflowData.blocks,
+      currentWorkflowData.edges,
+      currentWorkflowData.loops || {},
+      currentWorkflowData.parallels || {},
+      autoLayoutOptions
     )
 
-    const autoLayoutResult = await simAgentClient.makeRequest('/api/yaml/autolayout', {
-      body: {
-        workflowState,
-        options: autoLayoutOptions,
-        blockRegistry,
-        utilities: {
-          generateLoopBlocks: generateLoopBlocks.toString(),
-          generateParallelBlocks: generateParallelBlocks.toString(),
-          resolveOutputType: resolveOutputType.toString(),
-        },
-      },
-    })
-
-    // Log the full response for debugging
-    logger.info(`[${requestId}] Sim-agent autolayout response:`, {
-      success: autoLayoutResult.success,
-      status: autoLayoutResult.status,
-      error: autoLayoutResult.error,
-      hasData: !!autoLayoutResult.data,
-      hasWorkflowState: !!autoLayoutResult.data?.workflowState,
-      hasBlocks: !!autoLayoutResult.data?.blocks,
-      dataKeys: autoLayoutResult.data ? Object.keys(autoLayoutResult.data) : [],
-    })
-
-    if (
-      !autoLayoutResult.success ||
-      (!autoLayoutResult.data?.workflowState && !autoLayoutResult.data?.blocks)
-    ) {
+    if (!layoutResult.success || !layoutResult.blocks) {
       logger.error(`[${requestId}] Auto layout failed:`, {
-        success: autoLayoutResult.success,
-        error: autoLayoutResult.error,
-        status: autoLayoutResult.status,
-        fullResponse: autoLayoutResult,
-      })
-      const errorMessage =
-        autoLayoutResult.error ||
-        (autoLayoutResult.status === 401
-          ? 'Unauthorized - check API key'
-          : autoLayoutResult.status === 404
-            ? 'Sim-agent service not found'
-            : `HTTP ${autoLayoutResult.status}`)
-
-      return NextResponse.json(
-        {
-          error: 'Auto layout failed',
-          details: errorMessage,
-        },
-        { status: 500 }
-      )
-    }
-
-    // Handle both response formats from sim-agent
-    const layoutedBlocks =
-      autoLayoutResult.data?.workflowState?.blocks || autoLayoutResult.data?.blocks
-
-    if (!layoutedBlocks) {
-      logger.error(`[${requestId}] No blocks returned from sim-agent:`, {
-        responseData: autoLayoutResult.data,
+        error: layoutResult.error,
       })
       return NextResponse.json(
         {
           error: 'Auto layout failed',
-          details: 'No blocks returned from sim-agent',
+          details: layoutResult.error || 'Unknown error',
         },
         { status: 500 }
       )
     }
 
     const elapsed = Date.now() - startTime
-    const blockCount = Object.keys(layoutedBlocks).length
+    const blockCount = Object.keys(layoutResult.blocks).length
 
     logger.info(`[${requestId}] Autolayout completed successfully in ${elapsed}ms`, {
       blockCount,
       strategy: layoutOptions.strategy,
       workflowId,
     })
 
-    // Return the layouted blocks to the frontend - let the store handle saving
     return NextResponse.json({
       success: true,
       message: `Autolayout applied successfully to ${blockCount} blocks`,
@@ -245,7 +164,7 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
         direction: layoutOptions.direction,
         blockCount,
         elapsed: `${elapsed}ms`,
-        layoutedBlocks: layoutedBlocks,
+        layoutedBlocks: layoutResult.blocks,
       },
     })
   } catch (error) {