feat: add anthropic models to evals

Author: deloreyj

.github/workflows/evals.yml

@@ -25,6 +25,8 @@ jobs:
           echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" > ./apps/workers-bindings/.dev.vars
           echo "DEV_CLOUDFLARE_API_TOKEN=${{ secrets.DEV_CLOUDFLARE_API_TOKEN }}" >> ./apps/sandbox-container/.dev.vars
           echo "DEV_CLOUDFLARE_API_TOKEN=${{ secrets.DEV_CLOUDFLARE_API_TOKEN }}" >> ./apps/workers-bindings/.dev.vars
+          echo "ANTHROPIC_KEY=${{ secrets.ANTHROPIC_KEY }}" >> ./apps/sandbox-container/.dev.vars
+          echo "ANTHROPIC_KEY=${{ secrets.ANTHROPIC_KEY }}" >> ./apps/workers-bindings/.dev.vars
       - name: Verify .dev.vars file
         run: |
           du -h ./apps/sandbox-container/.dev.vars

apps/workers-bindings/evals/kv_namespaces.eval.ts

@@ -64,7 +64,7 @@ eachModel('$modelName', ({ model }) => {
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
 			const { promptOutput, toolCalls } = await runTask(client, model, input)
-
+			console.log('toolCalls', toolCalls)
 			const toolCall = toolCalls.find(
 				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_update
 			)

packages/eval-tools/package.json

@@ -11,6 +11,7 @@
 		"bin": "bin"
 	},
 	"dependencies": {
+		"@ai-sdk/anthropic": "^1.2.11",
 		"@ai-sdk/openai": "1.3.20",
 		"@cloudflare/vitest-pool-workers": "0.8.14",
 		"agents": "0.0.67",

packages/eval-tools/src/test-models.ts

@@ -1,4 +1,7 @@
+import { createAnthropic } from '@ai-sdk/anthropic'
+import { AnthropicMessagesModelId } from '@ai-sdk/anthropic/internal'
 import { createOpenAI } from '@ai-sdk/openai'
+import { OpenAIChatModelId } from '@ai-sdk/openai/internal'
 import { env } from 'cloudflare:test'
 import { describe } from 'vitest'
 import { createWorkersAI } from 'workers-ai-provider'
@@ -13,7 +16,7 @@ type AiTextGenerationModels = Exclude<
 	value2key<AiModels, BaseAiTextToImage>
 >
 
-function getOpenAiModel(modelName: string) {
+function getOpenAiModel(modelName: OpenAIChatModelId) {
 	if (!env.OPENAI_API_KEY) {
 		throw new Error('No API token set!')
 	}
@@ -26,6 +29,19 @@ function getOpenAiModel(modelName: string) {
 	return { modelName, model, ai }
 }
 
+function getAnthropicModel(modelName: AnthropicMessagesModelId) {
+	if (!env.ANTHROPIC_KEY) {
+		throw new Error('No Anthropic key set!')
+	}
+	const ai = createAnthropic({
+		apiKey: env.ANTHROPIC_KEY,
+	})
+
+	const model = ai(modelName)
+
+	return { modelName, model, ai }
+}
+
 function getWorkersAiModel(modelName: AiTextGenerationModels) {
 	if (!env.AI) {
 		throw new Error('No AI binding provided!')
@@ -40,7 +56,8 @@ function getWorkersAiModel(modelName: AiTextGenerationModels) {
 export const eachModel = describe.each([
 	getOpenAiModel('gpt-4o'),
 	getOpenAiModel('gpt-4o-mini'),
-
+	getAnthropicModel('claude-3-5-sonnet-latest'),
+	getAnthropicModel('claude-3-7-sonnet-latest'),
 	// llama 3 is somewhat inconsistent
 	//getWorkersAiModel("@cf/meta/llama-3.3-70b-instruct-fp8-fast")
 	// Currently llama 4 is having issues with tool calling