Add basic evals setup

Author: cmsparks

apps/sandbox-container/evals/initialize.eval.ts

@@ -0,0 +1,22 @@
+import { describeEval } from "vitest-evals"
+import { eachModel } from "@repo/eval-tools/src/test-models"
+import { checkFactuality } from "@repo/eval-tools/src/scorers"
+import { ToolExecutionOptions, ToolSet, generateText, tool } from "ai"
+import { MCPClientManager } from "agents/mcp/client"
+import { runTask } from "./utils"
+
+eachModel("$modelName", ({ model }) => {
+    describeEval("Runs container initialize", {
+        data: async () => [
+            { 
+                input: "create and ping a container", 
+                expected: "The container_initialize tool was called and then the container_ping tool was called"
+            }
+        ],
+        task: async (input) => {
+            return await runTask(model, input)
+        },
+        scorers: [checkFactuality],
+        threshold: 1
+    })
+})

apps/sandbox-container/evals/utils.ts

@@ -0,0 +1,62 @@
+import { MCPClientManager } from "agents/mcp/client"
+import { LanguageModelV1, StepResult, ToolSet, generateText, streamText, tool } from "ai"
+import { jsonSchemaToZod, type JsonSchemaObject } from "@n8n/json-schema-to-zod";
+
+export async function runTask(model: LanguageModelV1, input: string) {
+    const clientManager = new MCPClientManager("test-client", "0.0.0")
+    await clientManager.connect("http://localhost:8787/sse")
+
+    const tools = clientManager.listTools()
+    const toolSet: ToolSet = tools.reduce((acc, v) => {
+        acc[v.name] = tool({
+            parameters: jsonSchemaToZod(v.inputSchema as JsonSchemaObject),
+            description: v.description,
+            execute: async (args, opts) => {
+                console.log(`executing tool ${v.name}`)
+                const res = await clientManager.callTool(v, args, { signal: opts.abortSignal })
+                console.log(res.toolResult)
+                return res.content
+            },
+        })
+        return acc
+    }, {} as ToolSet)
+
+    const res = streamText({
+        model,
+        system: "You are an assistant responsible for evaluating the results of calling various tools. Given the user's query, use the tools available to you to answer the question.",
+        tools: toolSet,
+        prompt: input,
+        maxRetries: 1,
+        maxSteps: 10,
+    })
+
+    for await (const part of res.fullStream) {
+
+    }
+
+    let messagesWithTools = ""
+    const messages = (await res.response).messages
+    for (const message of messages) {
+        console.log(message.content)
+        for (const messagePart of message.content) {
+            if (typeof messagePart === "string") {
+                messagesWithTools += `<message_content type="text">
+                    ${messagePart}
+                </message_content>`
+            } else if (messagePart.type === "tool-call") {
+                messagesWithTools += `<message_content type=${messagePart.type}>
+                    <tool_name>${messagePart.toolName}</tool_name>
+                    <tool_arguments>${JSON.stringify(messagePart.args)}</tool_arguments>
+                </message_content>`
+            } else if (messagePart.type === "text") {
+                messagesWithTools += `<message_content type="text">
+                ${messagePart.text}
+            </message_content>`
+            }
+        }
+    }
+
+    console.log(messagesWithTools)
+
+    return messagesWithTools
+}

apps/sandbox-container/package.json

@@ -8,26 +8,31 @@
 		"dev": "concurrently \"tsx container/index.ts\" \"wrangler dev --var \"ENVIRONMENT:dev\"\"",
 		"build": "docker build .",
 		"start": "wrangler dev",
-		"start:container": "tsx container/index.ts"
+		"start:container": "tsx container/index.ts",
+		"eval:dev": "concurrently \"npm run dev\" \"vitest --config vitest.config.evals.ts\""
 	},
 	"dependencies": {
 		"@cloudflare/workers-types": "^4.20250320.0",
 		"@hono/node-server": "^1.13.8",
 		"@hono/zod-validator": "^0.4.3",
 		"@modelcontextprotocol/sdk": "^1.7.0",
+		"@n8n/json-schema-to-zod": "^1.1.0",
+		"@repo/eval-tools": "workspace:*",
 		"@types/node": "^22.13.10",
-		"agents": "^0.0.42",
+		"agents": "^0.0.60",
 		"cron-schedule": "^5.0.4",
 		"esbuild": "^0.25.1",
 		"hono": "^4.7.5",
 		"mime": "^4.0.6",
 		"octokit": "^4.1.2",
 		"partyserver": "^0.0.65",
 		"tsx": "^4.19.3",
+		"vitest-evals": "^0.1.4",
 		"workers-mcp": "0.1.0-3",
 		"zod": "^3.24.2"
 	},
 	"devDependencies": {
+		"ai": "^4.3.6",
 		"concurrently": "^9.1.2",
 		"wrangler": "^4.9.1"
 	}

apps/sandbox-container/tsconfig.json

@@ -1,16 +1,5 @@
 {
-	"compilerOptions": {
-		"target": "ESNext",
-		"lib": ["ESNext", "DOM"],
-		"jsx": "react-jsx",
-		"module": "ESNext",
-		"moduleResolution": "bundler",
-		"types": ["@cloudflare/workers-types/2023-07-01"],
-		"noEmit": true,
-		"esModuleInterop": true,
-		"forceConsistentCasingInFileNames": true,
-		"strict": true,
-		"skipLibCheck": true
-	},
-	"include": ["server/**.ts", "shared/**.ts"]
+	"extends": "@repo/typescript-config/workers.json",
+	"include": ["*/**.ts", "./vitest.config.evals.ts"],
+	"exclude": ["container/**.ts"]
 }

apps/sandbox-container/vitest.config.evals.ts

@@ -0,0 +1,13 @@
+import { defineWorkersConfig } from "@cloudflare/vitest-pool-workers/config";
+
+export default defineWorkersConfig({
+    test: {
+      include: ["**/*.eval.?(c|m)[jt]s?(x)"],
+      poolOptions: {
+        workers: {
+          isolatedStorage: true,
+          wrangler: { configPath: "./wrangler.jsonc" },
+        },
+      },
+    },
+});