More progress

Author: cte

apps/web-evals/package.json

@@ -10,9 +10,6 @@
 		"start": "next start"
 	},
 	"dependencies": {
-		"@roo-code/evals": "workspace:^",
-		"@roo-code/ipc": "workspace:^",
-		"@roo-code/types": "workspace:^",
 		"@hookform/resolvers": "^4.1.3",
 		"@radix-ui/react-alert-dialog": "^1.1.7",
 		"@radix-ui/react-dialog": "^1.1.6",
@@ -26,6 +23,9 @@
 		"@radix-ui/react-slot": "^1.1.2",
 		"@radix-ui/react-tabs": "^1.1.3",
 		"@radix-ui/react-tooltip": "^1.1.8",
+		"@roo-code/evals": "workspace:^",
+		"@roo-code/ipc": "workspace:^",
+		"@roo-code/types": "workspace:^",
 		"@tanstack/react-query": "^5.69.0",
 		"class-variance-authority": "^0.7.1",
 		"clsx": "^2.1.1",
@@ -38,10 +38,10 @@
 		"ps-tree": "^1.2.0",
 		"react": "^18.3.1",
 		"react-dom": "^18.3.1",
-		"react-hook-form": "^7.54.2",
+		"react-hook-form": "^7.57.0",
 		"react-use": "^17.6.0",
-		"sonner": "^2.0.2",
-		"tailwind-merge": "^3.0.2",
+		"sonner": "^2.0.5",
+		"tailwind-merge": "^3.3.0",
 		"tailwindcss-animate": "^1.0.7",
 		"vaul": "^1.1.2",
 		"zod": "^3.24.2"

packages/evals/package.json

@@ -7,8 +7,9 @@
 	"scripts": {
 		"lint": "eslint src --ext=ts --max-warnings=0",
 		"check-types": "tsc --noEmit",
-		"test": "dotenvx run -f .env.test -- vitest run",
+		"_test": "dotenvx run -f .env.test -- vitest run",
 		"clean": "rimraf dist .turbo",
+		"cli": "dotenvx run -f .env.development -- tsx src/cli/index.ts",
 		"drizzle-kit": "dotenvx run -f .env.development -- tsx node_modules/drizzle-kit/bin.cjs",
 		"drizzle-kit:test": "dotenvx run -f .env.test -- tsx node_modules/drizzle-kit/bin.cjs",
 		"db:start": "docker compose up -d",
@@ -24,9 +25,9 @@
 		"@roo-code/ipc": "workspace:^",
 		"@roo-code/types": "workspace:^",
 		"better-sqlite3": "^11.10.0",
+		"cmd-ts": "^0.13.0",
 		"drizzle-orm": "^0.44.1",
-		"execa": "^9.5.2",
-		"gluegun": "^5.2.0",
+		"execa": "^9.6.0",
 		"node-ipc": "^12.0.0",
 		"p-map": "^7.0.3",
 		"p-wait-for": "^5.0.2",

packages/evals/scripts/setup.sh

@@ -323,8 +323,7 @@ if [[ ! -s .env ]]; then
 fi
 
 echo -n "🗄️ Syncing Roo Code evals database... "
-pnpm --filter @roo-code/evals db:push &>/dev/null || exit 1
-pnpm --filter @roo-code/evals db:enable-wal &>/dev/null || exit 1
+pnpm --filter @roo-code/evals db:push --force &>/dev/null || exit 1
 echo "✅ Done"
 
 if ! grep -q "OPENROUTER_API_KEY" .env; then

packages/evals/src/cli/index.ts

@@ -3,7 +3,7 @@ import * as path from "path"
 
 import pWaitFor from "p-wait-for"
 import { execa, parseCommandString } from "execa"
-import { build, GluegunToolbox } from "gluegun"
+import { command, run, number, positional } from "cmd-ts"
 import psTree from "ps-tree"
 
 import { RooCodeEventName, IpcOrigin, IpcMessageType, TaskCommandName } from "@roo-code/types"
@@ -20,7 +20,7 @@ import {
 	updateTaskMetrics,
 	createToolError,
 } from "../db/index.js"
-import { __dirname, extensionDevelopmentPath, exercisesPath, type ExerciseLanguage } from "../exercises/index.js"
+import { type ExerciseLanguage, exercisesPath } from "../exercises/index.js"
 
 type TaskResult = { success: boolean }
 type TaskPromise = Promise<TaskResult>
@@ -37,14 +37,7 @@ const testCommands: Record<ExerciseLanguage, { commands: string[]; timeout?: num
 	rust: { commands: ["cargo test"] }, // timeout 15s bash -c "cd '$dir' && cargo test > /dev/null 2>&1"
 }
 
-const run = async (toolbox: GluegunToolbox) => {
-	const { config } = toolbox
-	const id = config.runId ? Number(config.runId) : undefined
-
-	if (!id) {
-		throw new Error("Run ID is required.")
-	}
-
+const runEvals = async (id: number) => {
 	const run = await findRun(id)
 	const tasks = await getTasks(run.id)
 
@@ -425,43 +418,23 @@ const runUnitTest = async ({ task }: { task: Task }) => {
 }
 
 const main = async () => {
-	const cli = build()
-		.brand("cli")
-		.src(__dirname)
-		.help()
-		.version()
-		.command({
-			name: "run",
-			description: "Run an eval",
-			run: ({ config, parameters }) => {
-				config.language = parameters.first
-				config.exercise = parameters.second
-
-				if (parameters.options["runId"]) {
-					config.runId = parameters.options["runId"]
-				}
+	const result = await run(
+		command({
+			name: "cli",
+			description: "Execute an eval run.",
+			version: "0.0.0",
+			args: {
+				runId: positional({ type: number, displayName: "runId" }),
 			},
-		})
-		.defaultCommand()
-		.create()
-
-	const toolbox = await cli.run(process.argv)
-	const { command } = toolbox
-
-	switch (command?.name) {
-		case "run":
-			await run(toolbox)
-			break
-	}
+			handler: (args) => runEvals(args.runId),
+		}),
+		process.argv.slice(2),
+	)
 
+	console.log(result)
 	process.exit(0)
 }
 
-if (!fs.existsSync(extensionDevelopmentPath)) {
-	console.error(`"extensionDevelopmentPath" does not exist.`)
-	process.exit(1)
-}
-
 if (!fs.existsSync(exercisesPath)) {
 	console.error(
 		`Exercises do not exist at ${exercisesPath}. Please run "git clone https://github.com/RooCodeInc/Roo-Code-Evals.git evals".`,

packages/evals/src/db/migrations/0000_old_gorilla_man.sql renamed to packages/evals/src/db/migrations/0000_young_trauma.sql

@@ -1,5 +1,5 @@
 {
-	"id": "caee25bc-e9ae-4d17-8448-11b879c8b66d",
+	"id": "b50d5e6a-0f3f-4605-a5e7-9351711fc5e4",
 	"prevId": "00000000-0000-0000-0000-000000000000",
 	"version": "7",
 	"dialect": "postgresql",

packages/evals/src/db/migrations/meta/0000_snapshot.json

@@ -5,8 +5,8 @@
 		{
 			"idx": 0,
 			"version": "7",
-			"when": 1748933185613,
-			"tag": "0000_old_gorilla_man",
+			"when": 1748937674449,
+			"tag": "0000_young_trauma",
 			"breakpoints": true
 		}
 	]

packages/evals/src/db/migrations/meta/_journal.json

@@ -1,9 +1,9 @@
 import { pgTable, text, timestamp, integer, real, boolean, jsonb, uniqueIndex } from "drizzle-orm/pg-core"
 import { relations } from "drizzle-orm"
 
-import { type RooCodeSettings, ToolName, type ToolUsage, toolNames } from "@roo-code/types"
+import type { RooCodeSettings, ToolName, ToolUsage } from "@roo-code/types"
 
-import { type ExerciseLanguage, exerciseLanguages } from "../exercises/index.js"
+import type { ExerciseLanguage } from "../exercises/index.js"
 
 /**
  * runs
@@ -45,7 +45,7 @@ export const tasks = pgTable(
 			.references(() => runs.id)
 			.notNull(),
 		taskMetricsId: integer("task_metrics_id").references(() => taskMetrics.id),
-		language: text({ enum: exerciseLanguages }).notNull().$type<ExerciseLanguage>(),
+		language: text().notNull().$type<ExerciseLanguage>(),
 		exercise: text().notNull(),
 		passed: boolean(),
 		startedAt: timestamp("started_at"),
@@ -97,7 +97,7 @@ export const toolErrors = pgTable("toolErrors", {
 	id: integer().primaryKey().generatedAlwaysAsIdentity(),
 	runId: integer("run_id").references(() => runs.id),
 	taskId: integer("task_id").references(() => tasks.id),
-	toolName: text("tool_name", { enum: toolNames }).notNull().$type<ToolName>(),
+	toolName: text("tool_name").notNull().$type<ToolName>(),
 	error: text().notNull(),
 	createdAt: timestamp("created_at").notNull(),
 })

packages/evals/src/db/schema.ts

@@ -1,2 +1,25 @@
-export * from "./exercises.js"
-export * from "./paths.js"
+import * as path from "path"
+import * as fs from "fs/promises"
+import { fileURLToPath } from "url"
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export const exercisesPath = path.resolve(__dirname, "..", "..", "..", "..", "..", "evals")
+
+export const exerciseLanguages = ["go", "java", "javascript", "python", "rust"] as const
+
+export type ExerciseLanguage = (typeof exerciseLanguages)[number]
+
+const listDirectories = async (relativePath: string) => {
+	try {
+		const targetPath = path.resolve(__dirname, relativePath)
+		const entries = await fs.readdir(targetPath, { withFileTypes: true })
+		return entries.filter((entry) => entry.isDirectory() && !entry.name.startsWith(".")).map((entry) => entry.name)
+	} catch (error) {
+		console.error(`Error listing directories at ${relativePath}:`, error)
+		return []
+	}
+}
+
+export const getExercisesForLanguage = async (language: ExerciseLanguage) =>
+	listDirectories(path.join(exercisesPath, language))