feat: working gemini evals

Author: deloreyj

.vscode/launch.json

@@ -10,6 +10,32 @@
 			"attachExistingChildren": false,
 			"autoAttachChildProcesses": false,
 			"sourceMaps": true // works with or without this line
+		},
+		{
+			"type": "node",
+			"request": "launch",
+			"name": "Open inspector with Vitest",
+			"runtimeExecutable": "npm",
+			"runtimeArgs": ["run", "eval:dev"],
+			"console": "integratedTerminal",
+			"cwd": "${workspaceFolder}/apps/workers-bindings"
+		},
+		{
+			"name": "Attach to Workers Runtime",
+			"type": "node",
+			"request": "attach",
+			"port": 9229,
+			"cwd": "/",
+			"resolveSourceMapLocations": null,
+			"attachExistingChildren": false,
+			"autoAttachChildProcesses": false
+		}
+	],
+	"compounds": [
+		{
+			"name": "Debug Workers tests",
+			"configurations": ["Open inspector with Vitest", "Attach to Workers Runtime"],
+			"stopAll": true
 		}
 	]
 }

apps/workers-bindings/evals/kv_namespaces.eval.ts

@@ -19,7 +19,6 @@ eachModel('$modelName', ({ model }) => {
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
 			const { promptOutput, toolCalls } = await runTask(client, model, input)
-			console.log('Creating kv namespace', JSON.stringify(toolCalls, null, 2))
 			const toolCall = toolCalls.find(
 				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_create
 			)
@@ -41,7 +40,6 @@ eachModel('$modelName', ({ model }) => {
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
 			const { promptOutput, toolCalls } = await runTask(client, model, input)
-			console.log('Listing kv namespaces', JSON.stringify(toolCalls, null, 2))
 			const toolCall = toolCalls.find(
 				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespaces_list
 			)
@@ -57,14 +55,13 @@ eachModel('$modelName', ({ model }) => {
 		data: async () => [
 			{
 				input:
-					'Rename my Cloudflare KV Namespace called "my-test-namespace" to "my-new-test-namespace". Assume the namespace exists. No need to look it up.',
+					'Rename my Cloudflare KV Namespace with ID 1234 to "my-new-test-namespace".',
 				expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_update} tool should be called to rename the kv namespace.`,
 			},
 		],
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
 			const { promptOutput, toolCalls } = await runTask(client, model, input)
-			console.log('Renaming kv namespace', JSON.stringify(toolCalls, null, 2))
 			const toolCall = toolCalls.find(
 				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_update
 			)
@@ -79,14 +76,13 @@ eachModel('$modelName', ({ model }) => {
 	describeEval('Get Cloudflare KV Namespace Details', {
 		data: async () => [
 			{
-				input: 'Get details of my Cloudflare KV Namespace called "my-new-test-namespace".',
+				input: 'Get details of my Cloudflare KV Namespace with ID 1234.',
 				expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_get} tool should be called to retrieve the details of the kv namespace.`,
 			},
 		],
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
 			const { promptOutput, toolCalls } = await runTask(client, model, input)
-			console.log('Getting kv namespace details', JSON.stringify(toolCalls, null, 2))
 			const toolCall = toolCalls.find(
 				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_get
 			)
@@ -101,7 +97,7 @@ eachModel('$modelName', ({ model }) => {
 	describeEval('Delete Cloudflare KV Namespace', {
 		data: async () => [
 			{
-				input: 'Delete the "my-new-test-namespace" kv namespace.',
+				input: 'Delete the kv namespace with ID 1234.',
 				expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_delete} tool should be called to delete the kv namespace.`,
 			},
 		],

apps/workers-bindings/package.json

@@ -8,7 +8,7 @@
 		"deploy": "wrangler deploy",
 		"deploy:staging": "wrangler deploy --env staging",
 		"deploy:production": "wrangler deploy --env production",
-		"eval:dev": "start-server-and-test --expect 404 eval:server http://localhost:8977 'vitest kv_namespaces.eval.ts --testTimeout=60000 --config vitest.config.evals.ts'",
+		"eval:dev": "start-server-and-test --expect 404 eval:server http://localhost:8977 'vitest --testTimeout=60000 --config vitest.config.evals.ts --inspect=9229 --no-file-parallelism'",
 		"eval:server": "wrangler dev --var ENVIRONMENT:test --var DEV_DISABLE_OAUTH:true --var DEV_CLOUDFLARE_EMAIL:[email protected] --inspector-port 9230 --port 8977",
 		"eval:ci": "start-server-and-test --expect 404 eval:server http://localhost:8977 'vitest run --testTimeout=60000 --config vitest.config.evals.ts'",
 		"dev": "wrangler dev",

packages/eval-tools/src/test-models.ts

@@ -85,7 +85,7 @@ export const eachModel = describe.each([
 	getOpenAiModel('gpt-4o'),
 	getOpenAiModel('gpt-4o-mini'),
 	getAnthropicModel('claude-3-5-sonnet-20241022'),
-	getGeminiModel('gemini-2.5-pro-exp-03-25')
+	getGeminiModel('gemini-2.0-flash')
 	// llama 3 is somewhat inconsistent
 	//getWorkersAiModel("@cf/meta/llama-3.3-70b-instruct-fp8-fast")
 	// Currently llama 4 is having issues with tool calling