Control evals concurrency in web app

Author: cte

evals/apps/cli/src/index.ts

@@ -33,7 +33,6 @@ import { IpcServer, IpcClient } from "@evals/ipc"
 import { __dirname, extensionDevelopmentPath, exercisesPath } from "./paths.js"
 import { getExercises } from "./exercises.js"
 
-const maxConcurrency = 2
 const taskTimeLimit = 5 * 60 * 1_000
 
 const testCommands: Record<ExerciseLanguage, { commands: string[]; timeout?: number; cwd?: string }> = {
@@ -74,12 +73,14 @@ const run = async (toolbox: GluegunToolbox) => {
 				const exercises = getExercises()[language as ExerciseLanguage]
 
 				await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), {
-					concurrency: 10,
+					concurrency: run.concurrency,
 				})
 			}
 		} else if (exercise === "all") {
 			const exercises = getExercises()[language as ExerciseLanguage]
-			await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), { concurrency: 10 })
+			await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), {
+				concurrency: run.concurrency,
+			})
 		} else {
 			language = language || (await askLanguage(prompt))
 			exercise = exercise || (await askExercise(prompt, language))
@@ -140,7 +141,7 @@ const run = async (toolbox: GluegunToolbox) => {
 			}
 		})
 
-		if (runningPromises.length >= maxConcurrency) {
+		if (runningPromises.length >= run.concurrency) {
 			await Promise.race(runningPromises)
 		}
 	}

evals/apps/web/src/app/runs/[id]/run.tsx

@@ -13,7 +13,6 @@ import {
 	DrawerHeader,
 	DrawerTitle,
 	ScrollArea,
-	Separator,
 	Table,
 	TableBody,
 	TableCell,

evals/apps/web/src/app/runs/new/new-run.tsx

@@ -7,7 +7,7 @@ import { useForm, FormProvider } from "react-hook-form"
 import { zodResolver } from "@hookform/resolvers/zod"
 import fuzzysort from "fuzzysort"
 import { toast } from "sonner"
-import { X, Rocket, Check, ChevronsUpDown, HardDriveUpload, CircleCheck } from "lucide-react"
+import { X, Rocket, Check, ChevronsUpDown, HardDriveUpload, CircleCheck, Cpu } from "lucide-react"
 
 import { globalSettingsSchema, rooCodeDefaults } from "@evals/types"
 
@@ -39,6 +39,7 @@ import {
 	PopoverContent,
 	PopoverTrigger,
 	ScrollArea,
+	Slider,
 } from "@/components/ui"
 
 import { SettingsDiff } from "./settings-diff"
@@ -68,6 +69,7 @@ export function NewRun() {
 			suite: "full",
 			exercises: [],
 			settings: undefined,
+			concurrency: 10,
 		},
 	})
 
@@ -79,7 +81,7 @@ export function NewRun() {
 		formState: { isSubmitting },
 	} = form
 
-	const [model, suite, settings] = watch(["model", "suite", "settings"])
+	const [model, suite, settings, concurrency] = watch(["model", "suite", "settings", "concurrency"])
 
 	const onSubmit = useCallback(
 		async (data: FormValues) => {
@@ -280,6 +282,40 @@ export function NewRun() {
 						)}
 					/>
 
+					<FormField
+						control={form.control}
+						name="concurrency"
+						render={({ field }) => (
+							<FormItem>
+								<FormLabel className="flex items-center gap-2">
+									<Cpu className="size-4" />
+									Concurrency
+								</FormLabel>
+								<FormControl>
+									<div className="flex flex-col gap-2">
+										<Slider
+											defaultValue={[field.value]}
+											min={1}
+											max={50}
+											step={1}
+											onValueChange={(value) => field.onChange(value[0])}
+										/>
+										<div className="flex justify-between text-xs text-muted-foreground">
+											<span>1</span>
+											<span>{field.value}</span>
+											<span>50</span>
+										</div>
+									</div>
+								</FormControl>
+								<FormDescription>
+									Number of tasks to run in parallel. Higher values may improve speed but increase
+									resource usage.
+								</FormDescription>
+								<FormMessage />
+							</FormItem>
+						)}
+					/>
+
 					<FormField
 						control={form.control}
 						name="description"

evals/apps/web/src/lib/schemas.ts

@@ -13,6 +13,7 @@ export const createRunSchema = z
 		suite: z.enum(["full", "partial"]),
 		exercises: z.array(z.string()).optional(),
 		settings: globalSettingsSchema.optional(),
+		concurrency: z.number().int().min(1).max(10).default(2),
 	})
 	.refine((data) => data.suite === "full" || (data.exercises || []).length > 0, {
 		message: "Exercises are required when running a partial suite.",

evals/packages/db/drizzle/0002_white_flatman.sql

@@ -0,0 +1 @@
+ALTER TABLE `runs` ADD `concurrency` integer DEFAULT 2 NOT NULL;