feat: add configurable timeout for evals (5-10 min)

- Add timeout field to CreateRun schema with min 5, max 10, default 5
- Add timeout slider UI component to /runs/new page
- Update database schema to include timeout column in runs table
- Create migration to add timeout column with default value of 5
- Update runTask.ts to use configurable timeout from run settings
- Pass timeout parameter through the createRun action

Author: roomote

apps/web-evals/src/actions/runs.ts

@@ -22,9 +22,10 @@ import { CreateRun } from "@/lib/schemas"
 const EVALS_REPO_PATH = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../../../evals")
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export async function createRun({ suite, exercises = [], systemPrompt, ...values }: CreateRun) {
+export async function createRun({ suite, exercises = [], systemPrompt, timeout, ...values }: CreateRun) {
 	const run = await _createRun({
 		...values,
+		timeout,
 		socketPath: "", // TODO: Get rid of this.
 	})
 

apps/web-evals/src/app/runs/new/new-run.tsx

@@ -21,6 +21,9 @@ import {
 	CONCURRENCY_MIN,
 	CONCURRENCY_MAX,
 	CONCURRENCY_DEFAULT,
+	TIMEOUT_MIN,
+	TIMEOUT_MAX,
+	TIMEOUT_DEFAULT,
 } from "@/lib/schemas"
 import { cn } from "@/lib/utils"
 import { useOpenRouterModels } from "@/hooks/use-open-router-models"
@@ -77,6 +80,7 @@ export function NewRun() {
 			exercises: [],
 			settings: undefined,
 			concurrency: CONCURRENCY_DEFAULT,
+			timeout: TIMEOUT_DEFAULT,
 		},
 	})
 
@@ -341,6 +345,29 @@ export function NewRun() {
 						)}
 					/>
 
+					<FormField
+						control={form.control}
+						name="timeout"
+						render={({ field }) => (
+							<FormItem>
+								<FormLabel>Timeout (minutes)</FormLabel>
+								<FormControl>
+									<div className="flex flex-row items-center gap-2">
+										<Slider
+											defaultValue={[field.value]}
+											min={TIMEOUT_MIN}
+											max={TIMEOUT_MAX}
+											step={1}
+											onValueChange={(value) => field.onChange(value[0])}
+										/>
+										<div>{field.value} min</div>
+									</div>
+								</FormControl>
+								<FormMessage />
+							</FormItem>
+						)}
+					/>
+
 					<FormField
 						control={form.control}
 						name="description"

apps/web-evals/src/lib/schemas.ts

@@ -12,6 +12,10 @@ export const CONCURRENCY_MIN = 1
 export const CONCURRENCY_MAX = 25
 export const CONCURRENCY_DEFAULT = 1
 
+export const TIMEOUT_MIN = 5
+export const TIMEOUT_MAX = 10
+export const TIMEOUT_DEFAULT = 5
+
 export const createRunSchema = z
 	.object({
 		model: z.string().min(1, { message: "Model is required." }),
@@ -20,6 +24,7 @@ export const createRunSchema = z
 		exercises: z.array(z.string()).optional(),
 		settings: rooCodeSettingsSchema.optional(),
 		concurrency: z.number().int().min(CONCURRENCY_MIN).max(CONCURRENCY_MAX),
+		timeout: z.number().int().min(TIMEOUT_MIN).max(TIMEOUT_MAX),
 		systemPrompt: z.string().optional(),
 	})
 	.refine((data) => data.suite === "full" || (data.exercises || []).length > 0, {

packages/evals/src/cli/runTask.ts

@@ -304,9 +304,10 @@ export const runTask = async ({ run, task, publish, logger }: RunTaskOptions) =>
 	})
 
 	try {
+		const timeoutMs = (run.timeout || 5) * 60 * 1_000 // Convert minutes to milliseconds
 		await pWaitFor(() => !!taskFinishedAt || !!taskAbortedAt || isClientDisconnected, {
 			interval: 1_000,
-			timeout: EVALS_TIMEOUT,
+			timeout: timeoutMs,
 		})
 	} catch (_error) {
 		taskTimedOut = true

packages/evals/src/db/migrations/0001_add_timeout_to_runs.sql

@@ -0,0 +1 @@
+ALTER TABLE "runs" ADD COLUMN "timeout" integer DEFAULT 5 NOT NULL;

packages/evals/src/db/schema.ts

@@ -18,6 +18,7 @@ export const runs = pgTable("runs", {
 	pid: integer(),
 	socketPath: text("socket_path").notNull(),
 	concurrency: integer().default(2).notNull(),
+	timeout: integer().default(5).notNull(),
 	passed: integer().default(0).notNull(),
 	failed: integer().default(0).notNull(),
 	createdAt: timestamp("created_at").notNull(),