Persist tool errors

Author: cte

evals/apps/cli/src/index.ts

@@ -29,6 +29,7 @@ import {
 	updateTask,
 	createTaskMetrics,
 	updateTaskMetrics,
+	createToolError,
 } from "@evals/db"
 import { IpcServer, IpcClient } from "@evals/ipc"
 
@@ -255,6 +256,12 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server
 			rooTaskId = payload[0]
 		}
 
+		if (eventName === RooCodeEventName.TaskToolFailed) {
+			// eslint-disable-next-line @typescript-eslint/no-unused-vars
+			const [_taskId, toolName, error] = payload
+			await createToolError({ taskId: task.id, toolName, error })
+		}
+
 		if (
 			(eventName === RooCodeEventName.TaskTokenUsageUpdated || eventName === RooCodeEventName.TaskCompleted) &&
 			taskMetricsId

evals/packages/db/src/index.ts

@@ -3,3 +3,4 @@ export * from "./schema.js"
 export * from "./queries/runs.js"
 export * from "./queries/tasks.js"
 export * from "./queries/taskMetrics.js"
+export * from "./queries/toolErrors.js"

evals/packages/db/src/queries/runs.ts

@@ -9,10 +9,8 @@ import { db } from "../db.js"
 import { createTaskMetrics } from "./taskMetrics.js"
 import { getTasks } from "./tasks.js"
 
-const table = schema.runs
-
 export const findRun = async (id: number) => {
-	const run = await db.query.runs.findFirst({ where: eq(table.id, id) })
+	const run = await db.query.runs.findFirst({ where: eq(schema.runs.id, id) })
 
 	if (!run) {
 		throw new RecordNotFoundError()
@@ -23,7 +21,7 @@ export const findRun = async (id: number) => {
 
 export const createRun = async (args: InsertRun) => {
 	const records = await db
-		.insert(table)
+		.insert(schema.runs)
 		.values({
 			...insertRunSchema.parse(args),
 			createdAt: new Date(),
@@ -40,7 +38,7 @@ export const createRun = async (args: InsertRun) => {
 }
 
 export const updateRun = async (id: number, values: UpdateRun) => {
-	const records = await db.update(table).set(values).where(eq(table.id, id)).returning()
+	const records = await db.update(schema.runs).set(values).where(eq(schema.runs.id, id)).returning()
 	const record = records[0]
 
 	if (!record) {
@@ -50,7 +48,8 @@ export const updateRun = async (id: number, values: UpdateRun) => {
 	return record
 }
 
-export const getRuns = async () => db.query.runs.findMany({ orderBy: desc(table.id), with: { taskMetrics: true } })
+export const getRuns = async () =>
+	db.query.runs.findMany({ orderBy: desc(schema.runs.id), with: { taskMetrics: true } })
 
 export const finishRun = async (runId: number) => {
 	const [values] = await db

evals/packages/db/src/queries/taskMetrics.ts

@@ -1,14 +1,12 @@
-import { eq, avg, min, max, and, isNotNull } from "drizzle-orm"
+import { eq } from "drizzle-orm"
 
 import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js"
 import type { InsertTaskMetrics, UpdateTaskMetrics } from "../schema.js"
-import { insertTaskMetricsSchema, taskMetrics, tasks, runs } from "../schema.js"
+import { insertTaskMetricsSchema, taskMetrics } from "../schema.js"
 import { db } from "../db.js"
 
-const table = taskMetrics
-
 export const findTaskMetrics = async (id: number) => {
-	const run = await db.query.taskMetrics.findFirst({ where: eq(table.id, id) })
+	const run = await db.query.taskMetrics.findFirst({ where: eq(taskMetrics.id, id) })
 
 	if (!run) {
 		throw new RecordNotFoundError()
@@ -19,7 +17,7 @@ export const findTaskMetrics = async (id: number) => {
 
 export const createTaskMetrics = async (args: InsertTaskMetrics) => {
 	const records = await db
-		.insert(table)
+		.insert(taskMetrics)
 		.values({
 			...insertTaskMetricsSchema.parse(args),
 			createdAt: new Date(),
@@ -36,7 +34,7 @@ export const createTaskMetrics = async (args: InsertTaskMetrics) => {
 }
 
 export const updateTaskMetrics = async (id: number, values: UpdateTaskMetrics) => {
-	const records = await db.update(table).set(values).where(eq(table.id, id)).returning()
+	const records = await db.update(taskMetrics).set(values).where(eq(taskMetrics.id, id)).returning()
 	const record = records[0]
 
 	if (!record) {
@@ -45,18 +43,3 @@ export const updateTaskMetrics = async (id: number, values: UpdateTaskMetrics) =
 
 	return record
 }
-
-export const successfulTaskDurations = async () => {
-	return db
-		.select({
-			runId: tasks.runId,
-			avgDuration: avg(taskMetrics.duration).mapWith(Number),
-			minDuration: min(taskMetrics.duration).mapWith(Number),
-			maxDuration: max(taskMetrics.duration).mapWith(Number),
-		})
-		.from(tasks)
-		.innerJoin(taskMetrics, eq(tasks.taskMetricsId, taskMetrics.id))
-		.innerJoin(runs, eq(tasks.runId, runs.id))
-		.where(and(eq(tasks.passed, true), isNotNull(runs.taskMetricsId)))
-		.groupBy(tasks.runId)
-}

evals/packages/db/src/queries/tasks.ts

@@ -7,10 +7,8 @@ import type { InsertTask, UpdateTask } from "../schema.js"
 import { insertTaskSchema, tasks } from "../schema.js"
 import { db } from "../db.js"
 
-const table = tasks
-
 export const findTask = async (id: number) => {
-	const run = await db.query.tasks.findFirst({ where: eq(table.id, id) })
+	const run = await db.query.tasks.findFirst({ where: eq(tasks.id, id) })
 
 	if (!run) {
 		throw new RecordNotFoundError()
@@ -21,7 +19,7 @@ export const findTask = async (id: number) => {
 
 export const createTask = async (args: InsertTask) => {
 	const records = await db
-		.insert(table)
+		.insert(tasks)
 		.values({
 			...insertTaskSchema.parse(args),
 			createdAt: new Date(),
@@ -38,7 +36,7 @@ export const createTask = async (args: InsertTask) => {
 }
 
 export const updateTask = async (id: number, values: UpdateTask) => {
-	const records = await db.update(table).set(values).where(eq(table.id, id)).returning()
+	const records = await db.update(tasks).set(values).where(eq(tasks.id, id)).returning()
 	const record = records[0]
 
 	if (!record) {
@@ -56,8 +54,8 @@ type GetTask = {
 
 export const getTask = async ({ runId, language, exercise }: GetTask) =>
 	db.query.tasks.findFirst({
-		where: and(eq(table.runId, runId), eq(table.language, language), eq(table.exercise, exercise)),
+		where: and(eq(tasks.runId, runId), eq(tasks.language, language), eq(tasks.exercise, exercise)),
 	})
 
 export const getTasks = async (runId: number) =>
-	db.query.tasks.findMany({ where: eq(table.runId, runId), with: { taskMetrics: true } })
+	db.query.tasks.findMany({ where: eq(tasks.runId, runId), with: { taskMetrics: true } })

evals/packages/db/src/queries/toolErrors.ts

@@ -0,0 +1,22 @@
+import { RecordNotCreatedError } from "./errors.js"
+import type { InsertToolError } from "../schema.js"
+import { insertToolErrorSchema, toolErrors } from "../schema.js"
+import { db } from "../db.js"
+
+export const createToolError = async (args: InsertToolError) => {
+	const records = await db
+		.insert(toolErrors)
+		.values({
+			...insertToolErrorSchema.parse(args),
+			createdAt: new Date(),
+		})
+		.returning()
+
+	const record = records[0]
+
+	if (!record) {
+		throw new RecordNotCreatedError()
+	}
+
+	return record
+}

evals/packages/db/src/schema.ts

@@ -123,6 +123,16 @@ export const toolErrorsRelations = relations(toolErrors, ({ one }) => ({
 	task: one(tasks, { fields: [toolErrors.taskId], references: [tasks.id] }),
 }))
 
+export type ToolError = typeof toolErrors.$inferSelect
+
+export const insertToolErrorSchema = createInsertSchema(toolErrors)
+	.omit({ id: true, createdAt: true })
+	.extend({ toolUsage: toolUsageSchema.optional() })
+
+export type InsertToolError = Omit<typeof toolErrors.$inferInsert, "id" | "createdAt">
+
+export type UpdateToolError = Partial<Omit<ToolError, "id" | "createdAt">>
+
 /**
  * schema
  */