Add MCP metrics (#69)

* Add basic MCP Server metrics

* Add CloudflareMCPServer wrapper to track metrics

* fix WAE metrics

* typecheck, lint, dependencies

* Fix bug with props not being initialized in constructor

* pr feedback

* fix package naming

Author: cmsparks

apps/docs-autorag/package.json

@@ -5,7 +5,7 @@
 	"scripts": {
 		"check:lint": "run-eslint-workers",
 		"check:types": "run-tsc",
-		"deploy": "wrangler deploy",
+		"deploy": "run-wrangler-deploy",
 		"dev": "wrangler dev",
 		"start": "wrangler dev",
 		"cf-typegen": "wrangler types",
@@ -16,6 +16,7 @@
 		"@hono/zod-validator": "0.4.3",
 		"@modelcontextprotocol/sdk": "1.9.0",
 		"@repo/mcp-common": "workspace:*",
+		"@repo/mcp-observability": "workspace:*",
 		"agents": "0.0.62",
 		"cloudflare": "4.2.0",
 		"hono": "4.7.6",
@@ -25,6 +26,7 @@
 	"devDependencies": {
 		"@cloudflare/vitest-pool-workers": "0.8.14",
 		"@cloudflare/workers-types": "4.20250410.0",
+		"@types/node": "^22.14.1",
 		"prettier": "3.5.3",
 		"typescript": "5.5.4",
 		"vitest": "3.0.9",

apps/docs-autorag/src/index.ts

@@ -1,5 +1,7 @@
-import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
 import { McpAgent } from 'agents/mcp'
+import { env } from 'cloudflare:workers'
+
+import { CloudflareMCPServer } from '@repo/mcp-common/src/server'
 
 import { registerDocsTools } from './tools/docs'
 
@@ -9,9 +11,9 @@ export type Props = never
 export type State = never
 
 export class CloudflareDocumentationMCP extends McpAgent<Env, State, Props> {
-	server = new McpServer({
-		name: 'Remote MCP Server with Cloudflare Documentation',
-		version: '1.0.0',
+	server = new CloudflareMCPServer(undefined, env.MCP_METRICS, {
+		name: env.MCP_SERVER_NAME,
+		version: env.MCP_SERVER_VERSION,
 	})
 
 	constructor(

apps/docs-autorag/worker-configuration.d.ts

@@ -1,11 +1,14 @@
 /* eslint-disable */
-// Generated by Wrangler by running `wrangler types` (hash: f47f9f62d3a1ddb9823a379fdafce5ce)
-// Runtime types generated with workerd@1.20250417.0 2025-03-10 nodejs_compat
+// Generated by Wrangler by running `wrangler types` (hash: 23bfeb1ecdb06845b72c16f50728901d)
+// Runtime types generated with workerd@1.20250422.0 2025-03-10 nodejs_compat
 declare namespace Cloudflare {
 	interface Env {
 		ENVIRONMENT: "development" | "staging" | "production";
 		AUTORAG_NAME: "cloudflare-docs-autorag";
+		MCP_SERVER_NAME: "PLACEHOLDER";
+		MCP_SERVER_VERSION: "PLACEHOLDER";
 		MCP_OBJECT: DurableObjectNamespace<import("./src/index").CloudflareDocumentationMCP>;
+		MCP_METRICS: AnalyticsEngineDataset;
 		AI: Ai;
 	}
 }

apps/docs-autorag/wrangler.jsonc

@@ -30,13 +30,21 @@
 	},
 	"vars": {
 		"ENVIRONMENT": "development",
-		"AUTORAG_NAME": "cloudflare-docs-autorag"
+		"AUTORAG_NAME": "cloudflare-docs-autorag",
+		"MCP_SERVER_NAME": "PLACEHOLDER",
+		"MCP_SERVER_VERSION": "PLACEHOLDER"
 	},
 	"dev": {
 		"port": 8976
 	},
 	"workers_dev": false,
 	"preview_urls": false,
+	"analytics_engine_datasets": [
+		{
+			"binding": "MCP_METRICS",
+			"dataset": "mcp-metrics-dev"
+		}
+	],
 	"env": {
 		"staging": {
 			"name": "mcp-cloudflare-docs-autorag-staging",
@@ -54,10 +62,15 @@
 				"ENVIRONMENT": "staging",
 				"AUTORAG_NAME": "cloudflare-docs-autorag"
 			},
-
 			"ai": {
 				"binding": "AI"
-			}
+			},
+			"analytics_engine_datasets": [
+				{
+					"binding": "MCP_METRICS",
+					"dataset": "mcp-metrics-staging"
+				}
+			]
 		},
 		"production": {
 			"name": "mcp-cloudflare-docs-autorag-production",
@@ -77,7 +90,13 @@
 			},
 			"ai": {
 				"binding": "AI"
-			}
+			},
+			"analytics_engine_datasets": [
+				{
+					"binding": "MCP_METRICS",
+					"dataset": "mcp-metrics-production"
+				}
+			]
 		}
 	}
 }

apps/sandbox-container/evals/exec.eval.ts

@@ -52,5 +52,6 @@ eachModel('$modelName', ({ model }) => {
 		},
 		scorers: [checkFactuality],
 		threshold: 1,
+		timeout: 60000,
 	})
 })

apps/sandbox-container/evals/files.eval.ts

@@ -51,6 +51,7 @@ eachModel('$modelName', ({ model }) => {
 		},
 		scorers: [checkFactuality],
 		threshold: 1,
+		timeout: 60000,
 	})
 
 	describeEval('Runs container file delete', {
@@ -102,5 +103,6 @@ eachModel('$modelName', ({ model }) => {
 		},
 		scorers: [checkFactuality],
 		threshold: 1,
+		timeout: 60000,
 	})
 })

apps/sandbox-container/evals/initialize.eval.ts

@@ -21,5 +21,6 @@ eachModel('$modelName', ({ model }) => {
 		},
 		scorers: [checkFactuality],
 		threshold: 1,
+		timeout: 60000,
 	})
 })

apps/sandbox-container/package.json

@@ -13,9 +13,9 @@
 		"postinstall": "mkdir -p workdir",
 		"test": "vitest",
 		"types": "wrangler types",
-		"eval:dev": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest --config vitest.config.evals.ts'",
+		"eval:dev": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest --testTimeout=60000 --config vitest.config.evals.ts'",
 		"eval:server": "concurrently \"tsx container/index.ts\" \"wrangler dev --var \"ENVIRONMENT:test\"\"",
-		"eval:ci": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest run --config vitest.config.evals.ts'"
+		"eval:ci": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest run --testTimeout=60000 --config vitest.config.evals.ts'"
 	},
 	"dependencies": {
 		"@cloudflare/workers-oauth-provider": "0.0.2",
@@ -26,6 +26,7 @@
 		"@n8n/json-schema-to-zod": "^1.1.0",
 		"@repo/eval-tools": "workspace:*",
 		"@repo/mcp-common": "workspace:*",
+		"@repo/mcp-observability": "workspace:*",
 		"@types/node": "^22.13.10",
 		"agents": "^0.0.62",
 		"cron-schedule": "^5.0.4",

apps/sandbox-container/server/containerManager.ts

@@ -1,7 +1,5 @@
 import { DurableObject } from 'cloudflare:workers'
 
-import { Env } from './index'
-
 export class ContainerManager extends DurableObject<Env> {
 	constructor(
 		public ctx: DurableObjectState,

apps/sandbox-container/server/containerMcp.ts

@@ -1,6 +1,7 @@
-import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
 import { McpAgent } from 'agents/mcp'
 
+import { CloudflareMCPServer } from '@repo/mcp-common/src/server'
+
 import { OPEN_CONTAINER_PORT } from '../shared/consts'
 import { ExecParams, FilePathParam, FileWrite } from '../shared/schema'
 import { MAX_CONTAINERS, proxyFetch, startAndWaitForPort } from './containerHelpers'
@@ -9,16 +10,21 @@ import { BASE_INSTRUCTIONS } from './prompts'
 import { fileToBase64, stripProtocolFromFilePath } from './utils'
 
 import type { FileList } from '../shared/schema'
-import type { Env, Props } from '.'
+import type { Props } from '.'
+
+export class ContainerMcpAgent extends McpAgent<Env, {}, Props> {
+	_server: CloudflareMCPServer | undefined
+	set server(server: CloudflareMCPServer) {
+		this._server = server
+	}
+
+	get server(): CloudflareMCPServer {
+		if (!this._server) {
+			throw new Error('Tried to access server before it was initialized')
+		}
 
-export class ContainerMcpAgent extends McpAgent<Env, Props> {
-	server = new McpServer(
-		{
-			name: 'Container MCP Agent',
-			version: '1.0.0',
-		},
-		{ instructions: BASE_INSTRUCTIONS }
-	)
+		return this._server
+	}
 
 	constructor(
 		public ctx: DurableObjectState,
@@ -44,6 +50,17 @@ export class ContainerMcpAgent extends McpAgent<Env, Props> {
 	}
 
 	async init() {
+		this.props.user.id
+		this.server = new CloudflareMCPServer(
+			this.props.user.id,
+			this.env.MCP_METRICS,
+			{
+				name: this.env.MCP_SERVER_NAME,
+				version: this.env.MCP_SERVER_VERSION,
+			},
+			{ instructions: BASE_INSTRUCTIONS }
+		)
+
 		this.server.tool(
 			'container_initialize',
 			'Start or reset the container',