More powerful runner

Author: cte

.github/workflows/evals.yml

@@ -13,7 +13,7 @@ jobs:
     evals:
         # Run if triggered manually or if PR has 'evals' label.
         if: github.event_name == 'workflow_dispatch' || contains(github.event.label.name, 'evals')
-        runs-on: ubuntu-latest
+        runs-on: blacksmith-16vcpu-ubuntu-2404
         timeout-minutes: 30
 
         steps:

packages/evals/src/cli/index.ts

@@ -24,7 +24,7 @@ const main = async () => {
 
 				try {
 					if (ci) {
-						await runCi()
+						await runCi({ concurrency: 3, exercisesPerLanguage: 5 })
 					} else if (runId !== -1) {
 						await runEvals(runId)
 					} else if (taskId !== -1) {

packages/evals/src/cli/runCi.ts

@@ -7,7 +7,7 @@ import { runEvals } from "./runEvals.js"
 
 export const runCi = async ({
 	concurrency = 1,
-	exercisesPerLanguage = 1,
+	exercisesPerLanguage,
 }: {
 	concurrency?: number
 	exercisesPerLanguage?: number
@@ -17,8 +17,13 @@ export const runCi = async ({
 	const run = await createRun({ model: "anthropic/claude-sonnet-4", socketPath: "", concurrency })
 
 	for (const language of exerciseLanguages) {
-		const exercises = (await getExercisesForLanguage(EVALS_REPO_PATH, language)).slice(0, exercisesPerLanguage)
-		await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }))
+		let exercises = await getExercisesForLanguage(EVALS_REPO_PATH, language)
+
+		if (exercisesPerLanguage) {
+			exercises = exercises.slice(0, exercisesPerLanguage)
+		}
+
+		await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), { concurrency })
 	}
 
 	await runEvals(run.id)