Record tool usages in the Cline object, and persist them in the db for evals (#2729)

Author: cte

evals/apps/cli/src/index.ts

@@ -275,7 +275,12 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server
 			})
 		}
 
-		if (eventName === RooCodeEventName.TaskCompleted || eventName === RooCodeEventName.TaskAborted) {
+		if (eventName === RooCodeEventName.TaskCompleted && taskMetricsId) {
+			const toolUsage = payload[2]
+			await updateTaskMetrics(taskMetricsId, { toolUsage })
+		}
+
+		if (eventName === RooCodeEventName.TaskAborted || eventName === RooCodeEventName.TaskCompleted) {
 			taskFinishedAt = Date.now()
 			await updateTask(task.id, { finishedAt: new Date() })
 		}

evals/apps/web/src/app/home.tsx

@@ -8,7 +8,7 @@ import { Ellipsis, Rocket } from "lucide-react"
 import type { Run, TaskMetrics } from "@evals/db"
 
 import { deleteRun } from "@/lib/server/runs"
-import { formatCurrency, formatDuration, formatTokens } from "@/lib"
+import { formatCurrency, formatDuration, formatTokens, formatToolUsageSuccessRate } from "@/lib/formatters"
 import {
 	Button,
 	Table,
@@ -59,7 +59,8 @@ export function Home({ runs }: { runs: (Run & { taskMetrics: TaskMetrics | null
 						<TableHead>Passed</TableHead>
 						<TableHead>Failed</TableHead>
 						<TableHead>% Correct</TableHead>
-						<TableHead className="text-center">Tokens In / Out</TableHead>
+						<TableHead>Tokens In / Out</TableHead>
+						<TableHead>Diff Edits</TableHead>
 						<TableHead>Cost</TableHead>
 						<TableHead>Duration</TableHead>
 						<TableHead />
@@ -79,12 +80,21 @@ export function Home({ runs }: { runs: (Run & { taskMetrics: TaskMetrics | null
 								</TableCell>
 								<TableCell>
 									{taskMetrics && (
-										<div className="flex items-center justify-evenly">
+										<div className="flex items-center gap-1.5">
 											<div>{formatTokens(taskMetrics.tokensIn)}</div>/
 											<div>{formatTokens(taskMetrics.tokensOut)}</div>
 										</div>
 									)}
 								</TableCell>
+								<TableCell>
+									{taskMetrics?.toolUsage?.apply_diff && (
+										<div className="flex flex-row items-center gap-1.5">
+											<div>{taskMetrics.toolUsage.apply_diff.attempts}</div>
+											<div>/</div>
+											<div>{formatToolUsageSuccessRate(taskMetrics.toolUsage.apply_diff)}</div>
+										</div>
+									)}
+								</TableCell>
 								<TableCell>{taskMetrics && formatCurrency(taskMetrics.cost)}</TableCell>
 								<TableCell>{taskMetrics && formatDuration(taskMetrics.duration)}</TableCell>
 								<TableCell>

evals/apps/web/src/app/runs/[id]/run.tsx

@@ -5,7 +5,7 @@ import { LoaderCircle } from "lucide-react"
 
 import * as db from "@evals/db"
 
-import { formatCurrency, formatDuration, formatTokens } from "@/lib"
+import { formatCurrency, formatDuration, formatTokens } from "@/lib/formatters"
 import { useRunStatus } from "@/hooks/use-run-status"
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
 

evals/apps/web/src/lib/format-currency.ts

@@ -0,0 +1,48 @@
+const formatter = new Intl.NumberFormat("en-US", {
+	style: "currency",
+	currency: "USD",
+})
+
+export const formatCurrency = (amount: number) => formatter.format(amount)
+
+export const formatDuration = (durationMs: number) => {
+	const seconds = Math.floor(durationMs / 1000)
+	const hours = Math.floor(seconds / 3600)
+	const minutes = Math.floor((seconds % 3600) / 60)
+	const remainingSeconds = seconds % 60
+
+	const parts = []
+
+	if (hours > 0) {
+		parts.push(`${hours}h`)
+	}
+
+	if (minutes > 0) {
+		parts.push(`${minutes}m`)
+	}
+
+	if (remainingSeconds > 0 || parts.length === 0) {
+		parts.push(`${remainingSeconds}s`)
+	}
+
+	return parts.join(" ")
+}
+
+export const formatTokens = (tokens: number) => {
+	if (tokens < 1000) {
+		return tokens.toString()
+	}
+
+	if (tokens < 1000000) {
+		return `${(tokens / 1000).toFixed(1)}k`
+	}
+
+	if (tokens < 1000000000) {
+		return `${(tokens / 1000000).toFixed(1)}M`
+	}
+
+	return `${(tokens / 1000000000).toFixed(1)}B`
+}
+
+export const formatToolUsageSuccessRate = (usage: { attempts: number; failures: number }) =>
+	usage.attempts === 0 ? '0%' : `${(((usage.attempts - usage.failures) / usage.attempts) * 100).toFixed(1)}%`

evals/apps/web/src/lib/format-duration.ts

@@ -0,0 +1 @@
+test.db

evals/apps/web/src/lib/format-tokens.ts

@@ -0,0 +1 @@
+ALTER TABLE `taskMetrics` ADD `toolUsage` text;