Add tests

Author: cte

evals/packages/db/.gitignore

@@ -0,0 +1 @@
+test.db

evals/packages/db/drizzle/0003_familiar_miss_america.sql

@@ -0,0 +1 @@
+ALTER TABLE `taskMetrics` ADD `toolUsage` text;

evals/packages/db/drizzle/0003_sweet_chimera.sql

@@ -1,7 +1,7 @@
 {
 	"version": "6",
 	"dialect": "sqlite",
-	"id": "a7a893e2-373a-4706-bcd4-772e2525db62",
+	"id": "61d48d20-f662-445d-9962-cf9cb165cbe7",
 	"prevId": "f49d9b0b-fda9-467a-9adb-c941d6cbf7ce",
 	"tables": {
 		"runs": {
@@ -165,7 +165,7 @@
 				},
 				"toolUsage": {
 					"name": "toolUsage",
-					"type": "blob",
+					"type": "text",
 					"primaryKey": false,
 					"notNull": false,
 					"autoincrement": false

evals/packages/db/drizzle/meta/0003_snapshot.json

@@ -26,8 +26,8 @@
 		{
 			"idx": 3,
 			"version": "6",
-			"when": 1744933023667,
-			"tag": "0003_familiar_miss_america",
+			"when": 1744950664129,
+			"tag": "0003_sweet_chimera",
 			"breakpoints": true
 		}
 	]

evals/packages/db/drizzle/meta/_journal.json

@@ -6,6 +6,7 @@
 	"scripts": {
 		"lint": "eslint src/**/*.ts --max-warnings=0",
 		"check-types": "tsc --noEmit",
+		"test": "vitest --globals --run",
 		"format": "prettier --write src",
 		"drizzle-kit": "dotenvx run -f ../../.env -- tsx node_modules/drizzle-kit/bin.cjs",
 		"db:generate": "pnpm drizzle-kit generate",
@@ -29,6 +30,8 @@
 	"devDependencies": {
 		"@evals/eslint-config": "workspace:^",
 		"@evals/typescript-config": "workspace:^",
-		"drizzle-kit": "^0.30.5"
+		"drizzle-kit": "^0.30.5",
+		"execa": "^9.5.2",
+		"vitest": "^3.0.9"
 	}
 }

evals/packages/db/package.json

@@ -0,0 +1,87 @@
+import { createRun, finishRun } from "../runs.js"
+import { createTask } from "../tasks.js"
+import { createTaskMetrics } from "../taskMetrics.js"
+
+describe("finishRun", async () => {
+	it("aggregates task metrics, including tool usage", async () => {
+		const run = await createRun({ model: "gpt-4.1-mini", socketPath: "/tmp/roo.sock" })
+
+		await createTask({
+			runId: run.id,
+			taskMetricsId: (
+				await createTaskMetrics({
+					duration: 45_000,
+					tokensIn: 100_000,
+					tokensOut: 2_000,
+					tokensContext: 102_000,
+					cacheWrites: 0,
+					cacheReads: 0,
+					cost: 0.05,
+					toolUsage: {
+						read_file: {
+							attempts: 3,
+							failures: 0,
+						},
+						apply_diff: {
+							attempts: 3,
+							failures: 1,
+						},
+					},
+				})
+			).id,
+			language: "go",
+			exercise: "go/say",
+			passed: true,
+			startedAt: new Date(),
+			finishedAt: new Date(),
+		})
+
+		await createTask({
+			runId: run.id,
+			taskMetricsId: (
+				await createTaskMetrics({
+					duration: 30_000,
+					tokensIn: 75_000,
+					tokensOut: 1_000,
+					tokensContext: 76_000,
+					cacheWrites: 0,
+					cacheReads: 0,
+					cost: 0.04,
+					toolUsage: {
+						read_file: {
+							attempts: 3,
+							failures: 0,
+						},
+						apply_diff: {
+							attempts: 2,
+							failures: 0,
+						},
+					},
+				})
+			).id,
+			language: "go",
+			exercise: "go/octal",
+			passed: true,
+			startedAt: new Date(),
+			finishedAt: new Date(),
+		})
+
+		const { taskMetrics } = await finishRun(run.id)
+
+		expect(taskMetrics).toEqual({
+			id: expect.any(Number),
+			tokensIn: 175000,
+			tokensOut: 3000,
+			tokensContext: 178000,
+			cacheWrites: 0,
+			cacheReads: 0,
+			cost: 0.09,
+			duration: 75000,
+			toolUsage: {
+				read_file: { attempts: 6, failures: 0 },
+				apply_diff: { attempts: 5, failures: 1 },
+			},
+			createdAt: expect.any(Date),
+		})
+	})
+})

evals/packages/db/src/queries/__tests__/runs.spec.ts

@@ -1,10 +1,13 @@
 import { desc, eq, inArray, sql, sum } from "drizzle-orm"
 
+import { ToolUsage } from "@evals/types"
+
 import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js"
 import type { InsertRun, UpdateRun } from "../schema.js"
 import { insertRunSchema, schema } from "../schema.js"
 import { db } from "../db.js"
 import { createTaskMetrics } from "./taskMetrics.js"
+import { getTasks } from "./tasks.js"
 
 const table = schema.runs
 
@@ -71,17 +74,30 @@ export const finishRun = async (runId: number) => {
 		throw new RecordNotFoundError()
 	}
 
+	const tasks = await getTasks(runId)
+
+	const toolUsage = tasks.reduce((acc, task) => {
+		Object.entries(task.taskMetrics?.toolUsage || {}).forEach(([key, { attempts, failures }]) => {
+			const tool = key as keyof ToolUsage
+			acc[tool] ??= { attempts: 0, failures: 0 }
+			acc[tool].attempts += attempts
+			acc[tool].failures += failures
+		})
+
+		return acc
+	}, {} as ToolUsage)
+
 	const { passed, failed, ...rest } = values
-	const taskMetrics = await createTaskMetrics(rest)
+	const taskMetrics = await createTaskMetrics({ ...rest, toolUsage })
 	await updateRun(runId, { taskMetricsId: taskMetrics.id, passed, failed })
 
-	const run = await db.query.runs.findFirst({ where: eq(table.id, runId), with: { taskMetrics: true } })
+	const run = await findRun(runId)
 
 	if (!run) {
 		throw new RecordNotFoundError()
 	}
 
-	return run
+	return { ...run, taskMetrics }
 }
 
 export const deleteRun = async (runId: number) => {

evals/packages/db/src/queries/runs.ts

@@ -84,7 +84,7 @@ export const taskMetrics = sqliteTable("taskMetrics", {
 	cacheReads: integer({ mode: "number" }).notNull(),
 	cost: real().notNull(),
 	duration: integer({ mode: "number" }).notNull(),
-	toolUsage: blob({ mode: "json" }).$type<ToolUsage>(),
+	toolUsage: text({ mode: "json" }).$type<ToolUsage>(),
 	createdAt: integer({ mode: "timestamp" }).notNull(),
 })
 

evals/packages/db/src/schema.ts

@@ -1,5 +1,8 @@
 {
 	"extends": "@evals/typescript-config/base.json",
+	"compilerOptions": {
+		"types": ["vitest/globals"]
+	},
 	"include": ["src"],
 	"exclude": ["node_modules"]
 }