fix(router): update router to handle azure creds the same way the agent block does (#2572)

* fix(router): update router to handle azure creds the same way the agent block does

* cleanup

Author: waleedlatif1

apps/sim/blocks/blocks/evaluator.ts

@@ -187,12 +187,16 @@ export const EvaluatorBlock: BlockConfig<EvaluatorResponse> = {
       type: 'combobox',
       placeholder: 'Type or select a model...',
       required: true,
+      defaultValue: 'claude-sonnet-4-5',
       options: () => {
         const providersState = useProvidersStore.getState()
         const baseModels = providersState.providers.base.models
         const ollamaModels = providersState.providers.ollama.models
+        const vllmModels = providersState.providers.vllm.models
         const openrouterModels = providersState.providers.openrouter.models
-        const allModels = Array.from(new Set([...baseModels, ...ollamaModels, ...openrouterModels]))
+        const allModels = Array.from(
+          new Set([...baseModels, ...ollamaModels, ...vllmModels, ...openrouterModels])
+        )
 
         return allModels.map((model) => {
           const icon = getProviderIcon(model)

apps/sim/blocks/blocks/router.ts

@@ -135,12 +135,16 @@ export const RouterBlock: BlockConfig<RouterResponse> = {
       type: 'combobox',
       placeholder: 'Type or select a model...',
       required: true,
+      defaultValue: 'claude-sonnet-4-5',
       options: () => {
         const providersState = useProvidersStore.getState()
         const baseModels = providersState.providers.base.models
         const ollamaModels = providersState.providers.ollama.models
+        const vllmModels = providersState.providers.vllm.models
         const openrouterModels = providersState.providers.openrouter.models
-        const allModels = Array.from(new Set([...baseModels, ...ollamaModels, ...openrouterModels]))
+        const allModels = Array.from(
+          new Set([...baseModels, ...ollamaModels, ...vllmModels, ...openrouterModels])
+        )
 
         return allModels.map((model) => {
           const icon = getProviderIcon(model)

apps/sim/executor/constants.ts

@@ -178,13 +178,13 @@ export const MEMORY = {
 } as const
 
 export const ROUTER = {
-  DEFAULT_MODEL: 'gpt-4o',
+  DEFAULT_MODEL: 'claude-sonnet-4-5',
   DEFAULT_TEMPERATURE: 0,
   INFERENCE_TEMPERATURE: 0.1,
 } as const
 
 export const EVALUATOR = {
-  DEFAULT_MODEL: 'gpt-4o',
+  DEFAULT_MODEL: 'claude-sonnet-4-5',
   DEFAULT_TEMPERATURE: 0.1,
   RESPONSE_SCHEMA_NAME: 'evaluation_response',
   JSON_INDENT: 2,

apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts

@@ -82,6 +82,7 @@ describe('EvaluatorBlockHandler', () => {
         { name: 'score2', description: 'Second score', range: { min: 0, max: 10 } },
       ],
       model: 'gpt-4o',
+      apiKey: 'test-api-key',
       temperature: 0.1,
     }
 
@@ -97,7 +98,6 @@ describe('EvaluatorBlockHandler', () => {
       })
     )
 
-    // Verify the request body contains the expected data
     const fetchCallArgs = mockFetch.mock.calls[0]
     const requestBody = JSON.parse(fetchCallArgs[1].body)
     expect(requestBody).toMatchObject({
@@ -137,6 +137,7 @@ describe('EvaluatorBlockHandler', () => {
     const inputs = {
       content: JSON.stringify(contentObj),
       metrics: [{ name: 'clarity', description: 'Clarity score', range: { min: 1, max: 5 } }],
+      apiKey: 'test-api-key',
     }
 
     mockFetch.mockImplementationOnce(() => {
@@ -169,6 +170,7 @@ describe('EvaluatorBlockHandler', () => {
       metrics: [
         { name: 'completeness', description: 'Data completeness', range: { min: 0, max: 1 } },
       ],
+      apiKey: 'test-api-key',
     }
 
     mockFetch.mockImplementationOnce(() => {
@@ -198,6 +200,7 @@ describe('EvaluatorBlockHandler', () => {
     const inputs = {
       content: 'Test content',
       metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      apiKey: 'test-api-key',
     }
 
     mockFetch.mockImplementationOnce(() => {
@@ -223,6 +226,7 @@ describe('EvaluatorBlockHandler', () => {
     const inputs = {
       content: 'Test content',
       metrics: [{ name: 'score', description: 'Score', range: { min: 0, max: 5 } }],
+      apiKey: 'test-api-key',
     }
 
     mockFetch.mockImplementationOnce(() => {
@@ -251,6 +255,7 @@ describe('EvaluatorBlockHandler', () => {
         { name: 'accuracy', description: 'Acc', range: { min: 0, max: 1 } },
         { name: 'fluency', description: 'Flu', range: { min: 0, max: 1 } },
       ],
+      apiKey: 'test-api-key',
     }
 
     mockFetch.mockImplementationOnce(() => {
@@ -276,6 +281,7 @@ describe('EvaluatorBlockHandler', () => {
     const inputs = {
       content: 'Test',
       metrics: [{ name: 'CamelCaseScore', description: 'Desc', range: { min: 0, max: 10 } }],
+      apiKey: 'test-api-key',
     }
 
     mockFetch.mockImplementationOnce(() => {
@@ -304,6 +310,7 @@ describe('EvaluatorBlockHandler', () => {
         { name: 'presentScore', description: 'Desc1', range: { min: 0, max: 5 } },
         { name: 'missingScore', description: 'Desc2', range: { min: 0, max: 5 } },
       ],
+      apiKey: 'test-api-key',
     }
 
     mockFetch.mockImplementationOnce(() => {
@@ -327,7 +334,7 @@ describe('EvaluatorBlockHandler', () => {
   })
 
   it('should handle server error responses', async () => {
-    const inputs = { content: 'Test error handling.' }
+    const inputs = { content: 'Test error handling.', apiKey: 'test-api-key' }
 
     // Override fetch mock to return an error
     mockFetch.mockImplementationOnce(() => {
@@ -340,4 +347,139 @@ describe('EvaluatorBlockHandler', () => {
 
     await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow('Server error')
   })
+
+  it('should handle Azure OpenAI models with endpoint and API version', async () => {
+    const inputs = {
+      content: 'Test content to evaluate',
+      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      model: 'gpt-4o',
+      apiKey: 'test-azure-key',
+      azureEndpoint: 'https://test.openai.azure.com',
+      azureApiVersion: '2024-07-01-preview',
+    }
+
+    mockGetProviderFromModel.mockReturnValue('azure-openai')
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ quality: 8 }),
+            model: 'gpt-4o',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+
+    expect(requestBody).toMatchObject({
+      provider: 'azure-openai',
+      model: 'gpt-4o',
+      apiKey: 'test-azure-key',
+      azureEndpoint: 'https://test.openai.azure.com',
+      azureApiVersion: '2024-07-01-preview',
+    })
+  })
+
+  it('should throw error when API key is missing for non-hosted models', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [{ name: 'score', description: 'Score', range: { min: 0, max: 10 } }],
+      model: 'gpt-4o',
+      // No apiKey provided
+    }
+
+    mockGetProviderFromModel.mockReturnValue('openai')
+
+    await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow(
+      /API key is required/
+    )
+  })
+
+  it('should handle Vertex AI models with OAuth credential', async () => {
+    const inputs = {
+      content: 'Test content to evaluate',
+      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      model: 'gemini-2.0-flash-exp',
+      vertexCredential: 'test-vertex-credential-id',
+      vertexProject: 'test-gcp-project',
+      vertexLocation: 'us-central1',
+    }
+
+    mockGetProviderFromModel.mockReturnValue('vertex')
+
+    // Mock the database query for Vertex credential
+    const mockDb = await import('@sim/db')
+    const mockAccount = {
+      id: 'test-vertex-credential-id',
+      accessToken: 'mock-access-token',
+      refreshToken: 'mock-refresh-token',
+      expiresAt: new Date(Date.now() + 3600000), // 1 hour from now
+    }
+    vi.spyOn(mockDb.db.query.account, 'findFirst').mockResolvedValue(mockAccount as any)
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ quality: 9 }),
+            model: 'gemini-2.0-flash-exp',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+
+    expect(requestBody).toMatchObject({
+      provider: 'vertex',
+      model: 'gemini-2.0-flash-exp',
+      vertexProject: 'test-gcp-project',
+      vertexLocation: 'us-central1',
+    })
+    expect(requestBody.apiKey).toBe('mock-access-token')
+  })
+
+  it('should use default model when not provided', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [{ name: 'score', description: 'Score', range: { min: 0, max: 10 } }],
+      apiKey: 'test-api-key',
+      // No model provided - should use default
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ score: 7 }),
+            model: 'claude-sonnet-4-5',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+
+    expect(requestBody.model).toBe('claude-sonnet-4-5')
+  })
 })

apps/sim/executor/handlers/evaluator/evaluator-handler.ts

@@ -8,7 +8,7 @@ import { BlockType, DEFAULTS, EVALUATOR, HTTP } from '@/executor/constants'
 import type { BlockHandler, ExecutionContext } from '@/executor/types'
 import { buildAPIUrl, extractAPIErrorMessage } from '@/executor/utils/http'
 import { isJSONString, parseJSON, stringifyJSON } from '@/executor/utils/json'
-import { calculateCost, getProviderFromModel } from '@/providers/utils'
+import { calculateCost, getApiKey, getProviderFromModel } from '@/providers/utils'
 import type { SerializedBlock } from '@/serializer/types'
 
 const logger = createLogger('EvaluatorBlockHandler')
@@ -35,9 +35,11 @@ export class EvaluatorBlockHandler implements BlockHandler {
     }
     const providerId = getProviderFromModel(evaluatorConfig.model)
 
-    let finalApiKey = evaluatorConfig.apiKey
+    let finalApiKey: string
     if (providerId === 'vertex' && evaluatorConfig.vertexCredential) {
       finalApiKey = await this.resolveVertexCredential(evaluatorConfig.vertexCredential)
+    } else {
+      finalApiKey = this.getApiKey(providerId, evaluatorConfig.model, evaluatorConfig.apiKey)
     }
 
     const processedContent = this.processContent(inputs.content)
@@ -122,6 +124,11 @@ export class EvaluatorBlockHandler implements BlockHandler {
         providerRequest.vertexLocation = evaluatorConfig.vertexLocation
       }
 
+      if (providerId === 'azure-openai') {
+        providerRequest.azureEndpoint = inputs.azureEndpoint
+        providerRequest.azureApiVersion = inputs.azureApiVersion
+      }
+
       const response = await fetch(url.toString(), {
         method: 'POST',
         headers: {
@@ -268,6 +275,20 @@ export class EvaluatorBlockHandler implements BlockHandler {
     return DEFAULTS.EXECUTION_TIME
   }
 
+  private getApiKey(providerId: string, model: string, inputApiKey: string): string {
+    try {
+      return getApiKey(providerId, model, inputApiKey)
+    } catch (error) {
+      logger.error('Failed to get API key:', {
+        provider: providerId,
+        model,
+        error: error instanceof Error ? error.message : String(error),
+        hasProvidedApiKey: !!inputApiKey,
+      })
+      throw new Error(error instanceof Error ? error.message : 'API key error')
+    }
+  }
+
   /**
    * Resolves a Vertex AI OAuth credential to an access token
    */