Skip to content

Commit 371231e

Browse files
deloreyjdeloreyj
committed
fix: make kv namespaces evals better
1 parent 599bfcf commit 371231e

File tree

4 files changed

+47
-22
lines changed

4 files changed

+47
-22
lines changed

‎apps/workers-bindings/evals/kv_namespaces.eval.ts‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,21 @@ import { checkFactuality } from '@repo/eval-tools/src/scorers'
55
import { eachModel } from '@repo/eval-tools/src/test-models'
66

77
import { initializeClient, runTask } from './utils' // Assuming utils.ts will exist here
8+
import { KV_NAMESPACE_TOOLS } from '@repo/mcp-common/src/tools/kv_namespace'
89

910
eachModel('$modelName', ({ model }) => {
1011
describeEval('Create Cloudflare KV Namespace', {
1112
data: async () => [
1213
{
1314
input: 'Create a new Cloudflare KV Namespace called "my-test-namespace".',
14-
expected: 'The kv_namespaces_create tool should be called to create a new kv namespace.',
15+
expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_create} tool should be called to create a new kv namespace.`,
1516
},
1617
],
1718
task: async (input: string) => {
1819
const client = await initializeClient(/* Pass necessary mocks/config */)
1920
const { promptOutput, toolCalls } = await runTask(client, model, input)
2021

21-
const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespace_create')
22+
const toolCall = toolCalls.find((call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_create)
2223
expect(toolCall, 'Tool kv_namespace_create was not called').toBeDefined()
2324

2425
return promptOutput
@@ -32,14 +33,14 @@ eachModel('$modelName', ({ model }) => {
3233
{
3334
input: 'List all my Cloudflare KV Namespaces.',
3435
expected:
35-
'The kv_namespaces_list tool should be called to retrieve the list of kv namespaces. There should be at least one kv namespace in the list.',
36+
`The ${KV_NAMESPACE_TOOLS.kv_namespaces_list} tool should be called to retrieve the list of kv namespaces. There should be at least one kv namespace in the list.`,
3637
},
3738
],
3839
task: async (input: string) => {
3940
const client = await initializeClient(/* Pass necessary mocks/config */)
4041
const { promptOutput, toolCalls } = await runTask(client, model, input)
4142

42-
const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespaces_list')
43+
const toolCall = toolCalls.find((call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespaces_list)
4344
expect(toolCall, 'Tool kv_namespaces_list was not called').toBeDefined()
4445

4546
return promptOutput
@@ -53,14 +54,14 @@ eachModel('$modelName', ({ model }) => {
5354
{
5455
input:
5556
'Rename my Cloudflare KV Namespace called "my-test-namespace" to "my-new-test-namespace".',
56-
expected: 'The kv_namespace_update tool should be called to rename the kv namespace.',
57+
expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_update} tool should be called to rename the kv namespace.`,
5758
},
5859
],
5960
task: async (input: string) => {
6061
const client = await initializeClient(/* Pass necessary mocks/config */)
6162
const { promptOutput, toolCalls } = await runTask(client, model, input)
6263

63-
const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespace_update')
64+
const toolCall = toolCalls.find((call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_update)
6465
expect(toolCall, 'Tool kv_namespace_update was not called').toBeDefined()
6566

6667
return promptOutput
@@ -74,15 +75,14 @@ eachModel('$modelName', ({ model }) => {
7475
{
7576
input: 'Get details of my Cloudflare KV Namespace called "my-new-test-namespace".',
7677
expected:
77-
'The kv_namespace_get tool should be called to retrieve the details of the kv namespace.',
78+
`The ${KV_NAMESPACE_TOOLS.kv_namespace_get} tool should be called to retrieve the details of the kv namespace.`,
7879
},
7980
],
8081
task: async (input: string) => {
8182
const client = await initializeClient(/* Pass necessary mocks/config */)
82-
const { promptOutput, toolCalls, fullResult } = await runTask(client, model, input)
83+
const { promptOutput, toolCalls } = await runTask(client, model, input)
8384

84-
console.log('fullResult', JSON.stringify(await fullResult.response, null, 2))
85-
const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespace_get')
85+
const toolCall = toolCalls.find((call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_get)
8686
expect(toolCall, 'Tool kv_namespace_get was not called').toBeDefined()
8787

8888
return promptOutput
@@ -95,14 +95,14 @@ eachModel('$modelName', ({ model }) => {
9595
data: async () => [
9696
{
9797
input: 'Look up the id of my only KV namespace and delete it.',
98-
expected: 'The kv_namespace_delete tool should be called to delete the kv namespace.',
98+
expected: `The ${KV_NAMESPACE_TOOLS.kv_namespace_delete} tool should be called to delete the kv namespace.`,
9999
},
100100
],
101101
task: async (input: string) => {
102102
const client = await initializeClient(/* Pass necessary mocks/config */)
103103
const { promptOutput, toolCalls } = await runTask(client, model, input)
104104

105-
const toolCall = toolCalls.find((call) => call.toolName === 'kv_namespace_delete')
105+
const toolCall = toolCalls.find((call) => call.toolName === KV_NAMESPACE_TOOLS.kv_namespace_delete)
106106
expect(toolCall, 'Tool kv_namespace_delete was not called').toBeDefined()
107107

108108
return promptOutput

‎apps/workers-bindings/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"deploy:staging": "wrangler deploy --env staging",
1010
"deploy:production": "wrangler deploy --env production",
1111
"eval:dev": "start-server-and-test --expect 404 eval:server http://localhost:8977 'vitest --testTimeout=60000 --config vitest.config.evals.ts'",
12-
"eval:server": "wrangler dev --var ENVIRONMENT:test --var DEV_DISABLE_OAUTH:true --var DEV_CLOUDFLARE_EMAIL:[email protected] --inspector-port 9230",
12+
"eval:server": "wrangler dev --var ENVIRONMENT:test --var DEV_DISABLE_OAUTH:true --var DEV_CLOUDFLARE_EMAIL:[email protected] --inspector-port 9230 --port 8977",
1313
"eval:ci": "start-server-and-test --expect 404 eval:server http://localhost:8977 'vitest run --testTimeout=60000 --config vitest.config.evals.ts'",
1414
"dev": "wrangler dev",
1515
"start": "wrangler dev",

‎apps/workers-bindings/wrangler.jsonc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
"enabled": true
3737
},
3838
"dev": {
39-
"port": 8977
39+
"port": 8976
4040
},
4141
"vars": {
4242
"ENVIRONMENT": "development",

‎packages/mcp-common/src/tools/kv_namespace.ts‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,27 @@ import {
77
KvNamespaceTitleSchema,
88
} from '../types/kv_namespace'
99

10+
export const KV_NAMESPACE_TOOLS = {
11+
kv_namespaces_list: 'kv_namespaces_list',
12+
kv_namespace_create: 'kv_namespace_create',
13+
kv_namespace_delete: 'kv_namespace_delete',
14+
kv_namespace_get: 'kv_namespace_get',
15+
kv_namespace_update: 'kv_namespace_update',
16+
}
17+
1018
export function registerKVTools(agent: CloudflareMcpAgent) {
1119
/**
1220
* Tool to list KV namespaces.
1321
*/
1422
agent.server.tool(
15-
'kv_namespaces_list',
16-
'List all of the kv namespaces in your Cloudflare account',
23+
KV_NAMESPACE_TOOLS.kv_namespaces_list,
24+
`
25+
List all of the kv namespaces in your Cloudflare account.
26+
Use this tool when you need to list all of the kv namespaces in your Cloudflare account.
27+
Returns a list of kv namespaces with the following properties:
28+
- id: The id of the kv namespace.
29+
- title: The title of the kv namespace.
30+
`,
1731
{ params: KvNamespacesListParamsSchema.optional() },
1832
async ({ params }) => {
1933
const account_id = await agent.getActiveAccountId()
@@ -27,7 +41,11 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
2741
...params,
2842
})
2943

30-
const namespaces = response.result ?? []
44+
let namespaces = response.result ?? []
45+
namespaces = namespaces.map(namespace => ({
46+
id: namespace.id,
47+
title: namespace.title,
48+
}))
3149

3250
return {
3351
content: [
@@ -57,7 +75,7 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
5775
* Tool to create a KV namespace.
5876
*/
5977
agent.server.tool(
60-
'kv_namespace_create',
78+
KV_NAMESPACE_TOOLS.kv_namespace_create,
6179
'Create a new kv namespace in your Cloudflare account',
6280
{
6381
title: KvNamespaceTitleSchema,
@@ -95,7 +113,7 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
95113
* Tool to delete a KV namespace.
96114
*/
97115
agent.server.tool(
98-
'kv_namespace_delete',
116+
KV_NAMESPACE_TOOLS.kv_namespace_delete,
99117
'Delete a kv namespace in your Cloudflare account',
100118
{
101119
namespace_id: KvNamespaceIdSchema,
@@ -133,8 +151,15 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
133151
* Tool to get details of a specific KV namespace.
134152
*/
135153
agent.server.tool(
136-
'kv_namespace_get',
137-
'Get details of a kv namespace in your Cloudflare account',
154+
KV_NAMESPACE_TOOLS.kv_namespace_get,
155+
`Get details of a kv namespace in your Cloudflare account.
156+
Use this tool when you need to get details of a specific kv namespace in your Cloudflare account.
157+
Returns a kv namespace with the following properties:
158+
- id: The id of the kv namespace.
159+
- title: The title of the kv namespace.
160+
- supports_url_encoding: Whether the kv namespace supports url encoding.
161+
- beta: Whether the kv namespace is in beta.
162+
`,
138163
{
139164
namespace_id: KvNamespaceIdSchema,
140165
},
@@ -171,7 +196,7 @@ export function registerKVTools(agent: CloudflareMcpAgent) {
171196
* Tool to update the title of a KV namespace.
172197
*/
173198
agent.server.tool(
174-
'kv_namespace_update',
199+
KV_NAMESPACE_TOOLS.kv_namespace_update,
175200
'Update the title of a kv namespace in your Cloudflare account',
176201
{
177202
namespace_id: KvNamespaceIdSchema,

0 commit comments

Comments
 (0)