Create pending tasks

Author: cte

benchmark/apps/cli/package.json

@@ -12,7 +12,7 @@
 		"@benchmark/db": "workspace:^",
 		"@vscode/test-electron": "^2.4.0",
 		"gluegun": "^5.1.2",
-		"p-map": "^7.0.3"
+		"uuid": "^11.1.0"
 	},
 	"devDependencies": {
 		"@benchmark/eslint-config": "workspace:^",

benchmark/apps/cli/src/exercises.ts

@@ -0,0 +1,31 @@
+import * as path from "path"
+import * as fs from "fs"
+
+import { filesystem } from "gluegun"
+
+import { type Language, languages } from "@benchmark/db"
+
+import { exercisesPath } from "./paths.js"
+
+let exercisesByLanguage: Record<Language, string[]> | null = null
+
+export const getExercises = () => {
+	if (exercisesByLanguage !== null) {
+		return exercisesByLanguage
+	}
+
+	const getLanguageExercises = (language: Language) =>
+		fs.existsSync(path.resolve(exercisesPath, language))
+			? filesystem
+					.subdirectories(path.resolve(exercisesPath, language))
+					.map((exercise) => path.basename(exercise))
+					.filter((exercise) => !exercise.startsWith("."))
+			: []
+
+	exercisesByLanguage = languages.reduce(
+		(collect, language) => ({ ...collect, [language]: getLanguageExercises(language) }),
+		{} as Record<Language, string[]>,
+	)
+
+	return exercisesByLanguage
+}

benchmark/apps/cli/src/index.ts

@@ -1,68 +1,90 @@
 import * as fs from "fs"
 import * as path from "path"
-import { fileURLToPath } from "url"
-import pMap from "p-map"
+import * as os from "os"
 
+import { v4 as uuidv4 } from "uuid"
 import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun"
 import { runTests } from "@vscode/test-electron"
 
-import { type Language, languages, type Run, findRun, createRun, getTask } from "@benchmark/db"
+import {
+	type Language,
+	languages,
+	type Run,
+	findRun,
+	createRun,
+	getPendingTask,
+	createPendingTask,
+	getTask,
+} from "@benchmark/db"
 
-const __dirname = path.dirname(fileURLToPath(import.meta.url))
-const extensionDevelopmentPath = path.resolve(__dirname, "..", "..", "..", "..")
-const extensionTestsPath = path.resolve(extensionDevelopmentPath, "benchmark/packages/runner/dist")
-const exercisesPath = path.resolve(extensionDevelopmentPath, "..", "exercises")
+import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js"
+import { getExercises } from "./exercises.js"
 
 export const isLanguage = (language: string): language is Language => languages.includes(language as Language)
 
 const run = async (toolbox: GluegunToolbox) => {
 	const { config, prompt } = toolbox
 	const id = config.runId ? Number(config.runId) : undefined
-	let { language, exercise } = config
+	const { language, exercise } = config
 
 	if (language === "all") {
-		const run = await findOrCreateRun({ id })
-		await runAll(run)
+		await runAll(id)
 	} else if (exercise === "all") {
-		const run = await findOrCreateRun({ id })
-		await runLanguage({ run, language })
+		await runLanguage({ id, language })
 	} else {
-		language = language || (await askLanguage(prompt))
-		exercise = exercise || (await askExercise(prompt, language))
-		const run = await findOrCreateRun({ id })
-		await runExercise({ run, language, exercise })
+		await runLanguageExercise({
+			id,
+			language: language || (await askLanguage(prompt)),
+			exercise: exercise || (await askExercise(prompt, language)),
+		})
 	}
 }
 
-const runAll = async (run: Run) =>
-	(await pMap(languages, (language) => runLanguage({ run, language }), { concurrency: 1 })).flatMap(
-		(language) => language,
-	)
+const runAll = async (id?: number) => {
+	const run = await findOrCreateRun({ id })
+	const exercises = getExercises()
 
-const runLanguage = async ({ run, language }: { run: Run; language: Language }) => {
-	const languagePath = path.resolve(exercisesPath, language)
+	for (const [language, languageExercises] of Object.entries(exercises)) {
+		await Promise.all(
+			languageExercises.map((exercise) =>
+				findOrCreatePendingTask({ runId: run.id, language: language as Language, exercise }),
+			),
+		)
+	}
 
-	if (!fs.existsSync(languagePath)) {
-		console.error(`Language directory ${languagePath} does not exist`)
-		process.exit(1)
+	for (const [language, languageExercises] of Object.entries(exercises)) {
+		for (const exercise of languageExercises) {
+			await runExercise({ run, language: language as Language, exercise })
+		}
 	}
+}
 
-	const exercises = filesystem
-		.subdirectories(languagePath)
-		.map((exercise) => path.basename(exercise))
-		.filter((exercise) => !exercise.startsWith("."))
-
-	const results = await pMap(
-		exercises,
-		async (exercise) => ({
-			language,
-			exercise,
-			result: await runExercise({ run, language, exercise }),
-		}),
-		{ concurrency: 1 },
+const runLanguage = async ({ id, language }: { id?: number; language: Language }) => {
+	const run = await findOrCreateRun({ id })
+	const exercises = getExercises()
+	const languageExercises = exercises[language]
+
+	await Promise.all(
+		languageExercises.map((exercise) => findOrCreatePendingTask({ runId: run.id, language, exercise })),
 	)
 
-	return results
+	for (const exercise of languageExercises) {
+		await runExercise({ run, language, exercise })
+	}
+}
+
+const runLanguageExercise = async ({
+	id,
+	language,
+	exercise,
+}: {
+	id?: number
+	language: Language
+	exercise: string
+}) => {
+	const run = await findOrCreateRun({ id })
+	await findOrCreatePendingTask({ runId: run.id, language, exercise })
+	return runExercise({ run, language, exercise })
 }
 
 const runExercise = async ({ run, language, exercise }: { run: Run; language: Language; exercise: string }) => {
@@ -127,10 +149,20 @@ const askExercise = async (prompt: GluegunPrompt, language: Language) => {
 	return exercise
 }
 
-type FindOrCreateRun = { id?: number; model?: string }
-
-const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: FindOrCreateRun) =>
-	id ? findRun(id) : createRun({ model })
+const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: { id?: number; model?: string }) =>
+	id
+		? findRun(id)
+		: createRun({ model, pid: process.pid, socketPath: path.resolve(os.tmpdir(), `benchmark-${uuidv4()}.sock`) })
+
+const findOrCreatePendingTask = async ({
+	runId,
+	language,
+	exercise,
+}: {
+	runId: number
+	language: Language
+	exercise: string
+}) => (await getPendingTask({ runId, language, exercise })) || (await createPendingTask({ runId, language, exercise }))
 
 const main = async () => {
 	const cli = build()

benchmark/apps/cli/src/paths.ts

@@ -0,0 +1,8 @@
+import * as path from "path"
+import { fileURLToPath } from "url"
+
+export const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export const extensionDevelopmentPath = path.resolve(__dirname, "..", "..", "..", "..")
+export const extensionTestsPath = path.resolve(extensionDevelopmentPath, "benchmark/packages/runner/dist")
+export const exercisesPath = path.resolve(extensionDevelopmentPath, "..", "exercises")

benchmark/packages/db/src/index.ts

@@ -15,3 +15,10 @@ export { findRun, createRun, getRuns } from "./queries/runs.js"
 
 export { type Task, type InsertTask, insertTaskSchema } from "./schema.js"
 export { findTask, createTask, getTask } from "./queries/tasks.js"
+
+/**
+ * pendingTasks
+ */
+
+export { type PendingTask, type InsertPendingTask, insertPendingTaskSchema } from "./schema.js"
+export { findPendingTask, createPendingTask, getPendingTask } from "./queries/pendingTasks.js"

benchmark/packages/db/src/queries/pendingTasks.ts

@@ -0,0 +1,50 @@
+import { and, eq } from "drizzle-orm"
+
+import { db } from "../db.js"
+import { type InsertPendingTask, insertPendingTaskSchema, pendingTasks } from "../schema.js"
+
+import { Language } from "../enums.js"
+import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js"
+
+export const findPendingTask = async (id: number) => {
+	const run = await db.query.pendingTasks.findFirst({ where: eq(pendingTasks.id, id) })
+
+	if (!run) {
+		throw new RecordNotFoundError()
+	}
+
+	return run
+}
+
+export const createPendingTask = async (args: InsertPendingTask) => {
+	const result = await db
+		.insert(pendingTasks)
+		.values({
+			...insertPendingTaskSchema.parse(args),
+			createdAt: new Date(),
+		})
+		.returning()
+
+	const task = result[0]
+
+	if (!task) {
+		throw new RecordNotCreatedError()
+	}
+
+	return task
+}
+
+type GetPendingTask = {
+	runId: number
+	language: Language
+	exercise: string
+}
+
+export const getPendingTask = async ({ runId, language, exercise }: GetPendingTask) =>
+	db.query.pendingTasks.findFirst({
+		where: and(
+			eq(pendingTasks.runId, runId),
+			eq(pendingTasks.language, language),
+			eq(pendingTasks.exercise, exercise),
+		),
+	})

benchmark/packages/db/src/schema.ts

@@ -90,4 +90,4 @@ export type InsertPendingTask = z.infer<typeof insertPendingTaskSchema>
  * schema
  */
 
-export const schema = { runs, tasks }
+export const schema = { runs, tasks, pendingTasks }