More tweaks

Author: cte

packages/evals/src/cli/processTask.ts

@@ -51,7 +51,7 @@ export const processTask = async ({ taskId, logger }: { taskId: number; logger?:
 export const processTaskInContainer = async ({
 	taskId,
 	logger,
-	maxRetries = 5,
+	maxRetries = 10,
 }: {
 	taskId: number
 	logger: FileLogger
@@ -68,8 +68,6 @@ export const processTaskInContainer = async ({
 	const command = `pnpm --filter @roo-code/evals cli --taskId ${taskId}`
 	logger.info(command)
 
-	let lastError: unknown
-
 	for (let attempt = 0; attempt <= maxRetries; attempt++) {
 		const containerName = `evals-task-${taskId}.${attempt}`
 		const args = [`--name ${containerName}`, ...baseArgs]
@@ -92,10 +90,8 @@ export const processTaskInContainer = async ({
 		try {
 			const result = await subprocess
 			logger.info(`container process completed with exit code: ${result.exitCode}`)
-			return result
+			return
 		} catch (error) {
-			lastError = error
-
 			if (error && typeof error === "object" && "exitCode" in error) {
 				logger.error(
 					`container process failed with exit code: ${error.exitCode} (attempt ${attempt + 1}/${maxRetries + 1})`,
@@ -111,5 +107,6 @@ export const processTaskInContainer = async ({
 	}
 
 	logger.error(`all ${maxRetries + 1} attempts failed, giving up`)
-	throw lastError
+
+	// TODO: Mark task as failed.
 }

packages/evals/src/cli/runEvals.ts

@@ -42,12 +42,17 @@ export const runEvals = async (runId: number) => {
 		await queue.addAll(
 			tasks
 				.filter((task) => task.finishedAt === null)
-				.map(
-					(task) => () =>
-						containerized
-							? processTaskInContainer({ taskId: task.id, logger })
-							: processTask({ taskId: task.id, logger }),
-				),
+				.map((task) => () => {
+					try {
+						if (containerized) {
+							processTaskInContainer({ taskId: task.id, logger })
+						} else {
+							processTask({ taskId: task.id, logger })
+						}
+					} catch (error) {
+						logger.error("error processing task", error)
+					}
+				}),
 		)
 
 		logger.info("finishRun")