fix: make kv namespaces evals better (#136)

Co-authored-by: [email protected] <[email protected]>

Author: deloreyj

apps/workers-bindings/evals/kv_namespaces.eval.ts

@@ -3,6 +3,7 @@ import { describeEval } from 'vitest-evals'
 
 import { checkFactuality } from '@repo/eval-tools/src/scorers'
 import { eachModel } from '@repo/eval-tools/src/test-models'
+import { KV_NAMESPACE_TOOLS } from '@repo/mcp-common/src/tools/kv_namespace'
 
 import { initializeClient, runTask } from './utils' // Assuming utils.ts will exist here
 
@@ -11,14 +12,16 @@ eachModel('$modelName', ({ model }) => {
 		data: async () => [
 			{
 				input: 'Create a new Cloudflare KV Namespace called "my-test-namespace".',
-				expected: 'The kv_namespaces_create tool should be called to create a new kv namespace.',
+				expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_create} tool should be called to create a new kv namespace.`,
 			},
 		],
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
 			const { promptOutput, toolCalls } = await runTask(client, model, input)
 
-			const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespace_create')
+			const toolCall = toolCalls.find(
+				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_create
+			)
 			expect(toolCall, 'Tool kv_namespace_create was not called').toBeDefined()
 
 			return promptOutput
@@ -31,15 +34,16 @@ eachModel('$modelName', ({ model }) => {
 		data: async () => [
 			{
 				input: 'List all my Cloudflare KV Namespaces.',
-				expected:
-					'The kv_namespaces_list tool should be called to retrieve the list of kv namespaces. There should be at least one kv namespace in the list.',
+				expected: `The ${KV_NAMESPACE_TOOLS.kv_namespaces_list} tool should be called to retrieve the list of kv namespaces. There should be at least one kv namespace in the list.`,
 			},
 		],
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
 			const { promptOutput, toolCalls } = await runTask(client, model, input)
 
-			const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespaces_list')
+			const toolCall = toolCalls.find(
+				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespaces_list
+			)
 			expect(toolCall, 'Tool kv_namespaces_list was not called').toBeDefined()
 
 			return promptOutput
@@ -53,14 +57,16 @@ eachModel('$modelName', ({ model }) => {
 			{
 				input:
 					'Rename my Cloudflare KV Namespace called "my-test-namespace" to "my-new-test-namespace".',
-				expected: 'The kv_namespace_update tool should be called to rename the kv namespace.',
+				expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_update} tool should be called to rename the kv namespace.`,
 			},
 		],
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
 			const { promptOutput, toolCalls } = await runTask(client, model, input)
 
-			const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespace_update')
+			const toolCall = toolCalls.find(
+				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_update
+			)
 			expect(toolCall, 'Tool kv_namespace_update was not called').toBeDefined()
 
 			return promptOutput
@@ -73,16 +79,16 @@ eachModel('$modelName', ({ model }) => {
 		data: async () => [
 			{
 				input: 'Get details of my Cloudflare KV Namespace called "my-new-test-namespace".',
-				expected:
-					'The kv_namespace_get tool should be called to retrieve the details of the kv namespace.',
+				expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_get} tool should be called to retrieve the details of the kv namespace.`,
 			},
 		],
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
-			const { promptOutput, toolCalls, fullResult } = await runTask(client, model, input)
+			const { promptOutput, toolCalls } = await runTask(client, model, input)
 
-			console.log('fullResult', JSON.stringify(await fullResult.response, null, 2))
-			const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespace_get')
+			const toolCall = toolCalls.find(
+				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_get
+			)
 			expect(toolCall, 'Tool kv_namespace_get was not called').toBeDefined()
 
 			return promptOutput
@@ -95,14 +101,16 @@ eachModel('$modelName', ({ model }) => {
 		data: async () => [
 			{
 				input: 'Look up the id of my only KV namespace and delete it.',
-				expected: 'The kv_namespace_delete tool should be called to delete the kv namespace.',
+				expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_delete} tool should be called to delete the kv namespace.`,
 			},
 		],
 		task: async (input: string) => {
 			const client = await initializeClient(/* Pass necessary mocks/config */)
 			const { promptOutput, toolCalls } = await runTask(client, model, input)
 
-			const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespace_delete')
+			const toolCall = toolCalls.find(
+				(call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_delete
+			)
 			expect(toolCall, 'Tool kv_namespace_delete was not called').toBeDefined()
 
 			return promptOutput

apps/workers-bindings/package.json

@@ -9,7 +9,7 @@
 		"deploy:staging": "wrangler deploy --env staging",
 		"deploy:production": "wrangler deploy --env production",
 		"eval:dev": "start-server-and-test --expect 404 eval:server http://localhost:8977 'vitest --testTimeout=60000 --config vitest.config.evals.ts'",
-		"eval:server": "wrangler dev --var ENVIRONMENT:test --var DEV_DISABLE_OAUTH:true --var DEV_CLOUDFLARE_EMAIL:[email protected] --inspector-port 9230",
+		"eval:server": "wrangler dev --var ENVIRONMENT:test --var DEV_DISABLE_OAUTH:true --var DEV_CLOUDFLARE_EMAIL:[email protected] --inspector-port 9230 --port 8977",
 		"eval:ci": "start-server-and-test --expect 404 eval:server http://localhost:8977 'vitest run --testTimeout=60000 --config vitest.config.evals.ts'",
 		"dev": "wrangler dev",
 		"start": "wrangler dev",

apps/workers-bindings/wrangler.jsonc

@@ -36,7 +36,7 @@
 		"enabled": true
 	},
 	"dev": {
-		"port": 8977
+		"port": 8976
 	},
 	"vars": {
 		"ENVIRONMENT": "development",

packages/mcp-common/src/tools/kv_namespace.ts

@@ -7,13 +7,27 @@ import {
 	KvNamespaceTitleSchema,
 } from '../types/kv_namespace'
 
+export const KV_NAMESPACE_TOOLS = {
+	kv_namespaces_list: 'kv_namespaces_list',
+	kv_namespace_create: 'kv_namespace_create',
+	kv_namespace_delete: 'kv_namespace_delete',
+	kv_namespace_get: 'kv_namespace_get',
+	kv_namespace_update: 'kv_namespace_update',
+}
+
 export function registerKVTools(agent: CloudflareMcpAgent) {
 	/**
 	 * Tool to list KV namespaces.
 	 */
 	agent.server.tool(
-		'kv_namespaces_list',
-		'List all of the kv namespaces in your Cloudflare account',
+		KV_NAMESPACE_TOOLS.kv_namespaces_list,
+		`
+			List all of the kv namespaces in your Cloudflare account. 
+			Use this tool when you need to list all of the kv namespaces in your Cloudflare account.
+			Returns a list of kv namespaces with the following properties:
+			- id: The id of the kv namespace.
+			- title: The title of the kv namespace.
+			`,
 		{ params: KvNamespacesListParamsSchema.optional() },
 		async ({ params }) => {
 			const account_id = await agent.getActiveAccountId()
@@ -27,7 +41,11 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
 					...params,
 				})
 
-				const namespaces = response.result ?? []
+				let namespaces = response.result ?? []
+				namespaces = namespaces.map((namespace) => ({
+					id: namespace.id,
+					title: namespace.title,
+				}))
 
 				return {
 					content: [
@@ -57,7 +75,7 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
 	 * Tool to create a KV namespace.
 	 */
 	agent.server.tool(
-		'kv_namespace_create',
+		KV_NAMESPACE_TOOLS.kv_namespace_create,
 		'Create a new kv namespace in your Cloudflare account',
 		{
 			title: KvNamespaceTitleSchema,
@@ -95,7 +113,7 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
 	 * Tool to delete a KV namespace.
 	 */
 	agent.server.tool(
-		'kv_namespace_delete',
+		KV_NAMESPACE_TOOLS.kv_namespace_delete,
 		'Delete a kv namespace in your Cloudflare account',
 		{
 			namespace_id: KvNamespaceIdSchema,
@@ -133,8 +151,15 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
 	 * Tool to get details of a specific KV namespace.
 	 */
 	agent.server.tool(
-		'kv_namespace_get',
-		'Get details of a kv namespace in your Cloudflare account',
+		KV_NAMESPACE_TOOLS.kv_namespace_get,
+		`Get details of a kv namespace in your Cloudflare account.
+		Use this tool when you need to get details of a specific kv namespace in your Cloudflare account.
+		Returns a kv namespace with the following properties:
+			- id: The id of the kv namespace.
+			- title: The title of the kv namespace.
+			- supports_url_encoding: Whether the kv namespace supports url encoding.
+			- beta: Whether the kv namespace is in beta.
+		`,
 		{
 			namespace_id: KvNamespaceIdSchema,
 		},
@@ -171,7 +196,7 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
 	 * Tool to update the title of a KV namespace.
 	 */
 	agent.server.tool(
-		'kv_namespace_update',
+		KV_NAMESPACE_TOOLS.kv_namespace_update,
 		'Update the title of a kv namespace in your Cloudflare account',
 		{
 			namespace_id: KvNamespaceIdSchema,