Skip to content

Commit 48ca0f2

Browse files
cmsparkscmsparks
authored
Add MCP metrics (#69)
* Add basic MCP Server metrics * Add CloudflareMCPServer wrapper to track metrics * fix WAE metrics * typecheck, lint, dependencies * Fix bug with props not being initialized in constructor * pr feedback * fix package naming
1 parent e25967e commit 48ca0f2

35 files changed

+5547
-5645
lines changed

‎apps/docs-autorag/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"scripts": {
66
"check:lint": "run-eslint-workers",
77
"check:types": "run-tsc",
8-
"deploy": "wrangler deploy",
8+
"deploy": "run-wrangler-deploy",
99
"dev": "wrangler dev",
1010
"start": "wrangler dev",
1111
"cf-typegen": "wrangler types",
@@ -16,6 +16,7 @@
1616
"@hono/zod-validator": "0.4.3",
1717
"@modelcontextprotocol/sdk": "1.9.0",
1818
"@repo/mcp-common": "workspace:*",
19+
"@repo/mcp-observability": "workspace:*",
1920
"agents": "0.0.62",
2021
"cloudflare": "4.2.0",
2122
"hono": "4.7.6",
@@ -25,6 +26,7 @@
2526
"devDependencies": {
2627
"@cloudflare/vitest-pool-workers": "0.8.14",
2728
"@cloudflare/workers-types": "4.20250410.0",
29+
"@types/node": "^22.14.1",
2830
"prettier": "3.5.3",
2931
"typescript": "5.5.4",
3032
"vitest": "3.0.9",

‎apps/docs-autorag/src/index.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
21
import { McpAgent } from 'agents/mcp'
2+
import { env } from 'cloudflare:workers'
3+
4+
import { CloudflareMCPServer } from '@repo/mcp-common/src/server'
35

46
import { registerDocsTools } from './tools/docs'
57

@@ -9,9 +11,9 @@ export type Props = never
911
export type State = never
1012

1113
export class CloudflareDocumentationMCP extends McpAgent<Env, State, Props> {
12-
server = new McpServer({
13-
name: 'Remote MCP Server with Cloudflare Documentation',
14-
version: '1.0.0',
14+
server = new CloudflareMCPServer(undefined, env.MCP_METRICS, {
15+
name: env.MCP_SERVER_NAME,
16+
version: env.MCP_SERVER_VERSION,
1517
})
1618

1719
constructor(

‎apps/docs-autorag/worker-configuration.d.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
/* eslint-disable */
2-
// Generated by Wrangler by running `wrangler types` (hash: f47f9f62d3a1ddb9823a379fdafce5ce)
3-
// Runtime types generated with workerd@1.20250417.0 2025-03-10 nodejs_compat
2+
// Generated by Wrangler by running `wrangler types` (hash: 23bfeb1ecdb06845b72c16f50728901d)
3+
// Runtime types generated with workerd@1.20250422.0 2025-03-10 nodejs_compat
44
declare namespace Cloudflare {
55
interface Env {
66
ENVIRONMENT: "development" | "staging" | "production";
77
AUTORAG_NAME: "cloudflare-docs-autorag";
8+
MCP_SERVER_NAME: "PLACEHOLDER";
9+
MCP_SERVER_VERSION: "PLACEHOLDER";
810
MCP_OBJECT: DurableObjectNamespace<import("./src/index").CloudflareDocumentationMCP>;
11+
MCP_METRICS: AnalyticsEngineDataset;
912
AI: Ai;
1013
}
1114
}

‎apps/docs-autorag/wrangler.jsonc‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,21 @@
3030
},
3131
"vars": {
3232
"ENVIRONMENT": "development",
33-
"AUTORAG_NAME": "cloudflare-docs-autorag"
33+
"AUTORAG_NAME": "cloudflare-docs-autorag",
34+
"MCP_SERVER_NAME": "PLACEHOLDER",
35+
"MCP_SERVER_VERSION": "PLACEHOLDER"
3436
},
3537
"dev": {
3638
"port": 8976
3739
},
3840
"workers_dev": false,
3941
"preview_urls": false,
42+
"analytics_engine_datasets": [
43+
{
44+
"binding": "MCP_METRICS",
45+
"dataset": "mcp-metrics-dev"
46+
}
47+
],
4048
"env": {
4149
"staging": {
4250
"name": "mcp-cloudflare-docs-autorag-staging",
@@ -54,10 +62,15 @@
5462
"ENVIRONMENT": "staging",
5563
"AUTORAG_NAME": "cloudflare-docs-autorag"
5664
},
57-
5865
"ai": {
5966
"binding": "AI"
60-
}
67+
},
68+
"analytics_engine_datasets": [
69+
{
70+
"binding": "MCP_METRICS",
71+
"dataset": "mcp-metrics-staging"
72+
}
73+
]
6174
},
6275
"production": {
6376
"name": "mcp-cloudflare-docs-autorag-production",
@@ -77,7 +90,13 @@
7790
},
7891
"ai": {
7992
"binding": "AI"
80-
}
93+
},
94+
"analytics_engine_datasets": [
95+
{
96+
"binding": "MCP_METRICS",
97+
"dataset": "mcp-metrics-production"
98+
}
99+
]
81100
}
82101
}
83102
}

‎apps/sandbox-container/evals/exec.eval.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,5 +52,6 @@ eachModel('$modelName', ({ model }) => {
5252
},
5353
scorers: [checkFactuality],
5454
threshold: 1,
55+
timeout: 60000,
5556
})
5657
})

‎apps/sandbox-container/evals/files.eval.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ eachModel('$modelName', ({ model }) => {
5151
},
5252
scorers: [checkFactuality],
5353
threshold: 1,
54+
timeout: 60000,
5455
})
5556

5657
describeEval('Runs container file delete', {
@@ -102,5 +103,6 @@ eachModel('$modelName', ({ model }) => {
102103
},
103104
scorers: [checkFactuality],
104105
threshold: 1,
106+
timeout: 60000,
105107
})
106108
})

‎apps/sandbox-container/evals/initialize.eval.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,6 @@ eachModel('$modelName', ({ model }) => {
2121
},
2222
scorers: [checkFactuality],
2323
threshold: 1,
24+
timeout: 60000,
2425
})
2526
})

‎apps/sandbox-container/package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@
1313
"postinstall": "mkdir -p workdir",
1414
"test": "vitest",
1515
"types": "wrangler types",
16-
"eval:dev": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest --config vitest.config.evals.ts'",
16+
"eval:dev": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest --testTimeout=60000 --config vitest.config.evals.ts'",
1717
"eval:server": "concurrently \"tsx container/index.ts\" \"wrangler dev --var \"ENVIRONMENT:test\"\"",
18-
"eval:ci": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest run --config vitest.config.evals.ts'"
18+
"eval:ci": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest run --testTimeout=60000 --config vitest.config.evals.ts'"
1919
},
2020
"dependencies": {
2121
"@cloudflare/workers-oauth-provider": "0.0.2",
@@ -26,6 +26,7 @@
2626
"@n8n/json-schema-to-zod": "^1.1.0",
2727
"@repo/eval-tools": "workspace:*",
2828
"@repo/mcp-common": "workspace:*",
29+
"@repo/mcp-observability": "workspace:*",
2930
"@types/node": "^22.13.10",
3031
"agents": "^0.0.62",
3132
"cron-schedule": "^5.0.4",

‎apps/sandbox-container/server/containerManager.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
import { DurableObject } from 'cloudflare:workers'
22

3-
import { Env } from './index'
4-
53
export class ContainerManager extends DurableObject<Env> {
64
constructor(
75
public ctx: DurableObjectState,

‎apps/sandbox-container/server/containerMcp.ts‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
21
import { McpAgent } from 'agents/mcp'
32

3+
import { CloudflareMCPServer } from '@repo/mcp-common/src/server'
4+
45
import { OPEN_CONTAINER_PORT } from '../shared/consts'
56
import { ExecParams, FilePathParam, FileWrite } from '../shared/schema'
67
import { MAX_CONTAINERS, proxyFetch, startAndWaitForPort } from './containerHelpers'
@@ -9,16 +10,21 @@ import { BASE_INSTRUCTIONS } from './prompts'
910
import { fileToBase64, stripProtocolFromFilePath } from './utils'
1011

1112
import type { FileList } from '../shared/schema'
12-
import type { Env, Props } from '.'
13+
import type { Props } from '.'
14+
15+
export class ContainerMcpAgent extends McpAgent<Env, {}, Props> {
16+
_server: CloudflareMCPServer | undefined
17+
set server(server: CloudflareMCPServer) {
18+
this._server = server
19+
}
20+
21+
get server(): CloudflareMCPServer {
22+
if (!this._server) {
23+
throw new Error('Tried to access server before it was initialized')
24+
}
1325

14-
export class ContainerMcpAgent extends McpAgent<Env, Props> {
15-
server = new McpServer(
16-
{
17-
name: 'Container MCP Agent',
18-
version: '1.0.0',
19-
},
20-
{ instructions: BASE_INSTRUCTIONS }
21-
)
26+
return this._server
27+
}
2228

2329
constructor(
2430
public ctx: DurableObjectState,
@@ -44,6 +50,17 @@ export class ContainerMcpAgent extends McpAgent<Env, Props> {
4450
}
4551

4652
async init() {
53+
this.props.user.id
54+
this.server = new CloudflareMCPServer(
55+
this.props.user.id,
56+
this.env.MCP_METRICS,
57+
{
58+
name: this.env.MCP_SERVER_NAME,
59+
version: this.env.MCP_SERVER_VERSION,
60+
},
61+
{ instructions: BASE_INSTRUCTIONS }
62+
)
63+
4764
this.server.tool(
4865
'container_initialize',
4966
'Start or reset the container',

0 commit comments

Comments
 (0)