diff --git a/evals/packages/db/package.json b/evals/packages/db/package.json index c140ffa0480..9e22267d221 100644 --- a/evals/packages/db/package.json +++ b/evals/packages/db/package.json @@ -15,7 +15,8 @@ "db:check": "pnpm drizzle-kit check", "db:up": "pnpm drizzle-kit up", "db:studio": "pnpm drizzle-kit studio", - "db:enable-wal": "dotenvx run -f ../../.env -- tsx scripts/enable-wal.mts" + "db:enable-wal": "dotenvx run -f ../../.env -- tsx scripts/enable-wal.mts", + "db:copy-run": "dotenvx run -f ../../.env -- tsx scripts/copy-run.mts" }, "dependencies": { "@evals/types": "workspace:^", diff --git a/evals/packages/db/scripts/copy-run.mts b/evals/packages/db/scripts/copy-run.mts new file mode 100644 index 00000000000..0beb97a8451 --- /dev/null +++ b/evals/packages/db/scripts/copy-run.mts @@ -0,0 +1,97 @@ +import { drizzle } from "drizzle-orm/libsql" +import { eq } from "drizzle-orm" + +import { db as sourceDb } from "../src/db.js" +import { schema } from "../src/schema.js" + +const copyRun = async (runId: number) => { + const destDb = drizzle({ + schema, + connection: { url: process.env.TURSO_CONNECTION_URL!, authToken: process.env.TURSO_AUTH_TOKEN! }, + }) + + const run = await sourceDb.query.runs.findFirst({ + where: eq(schema.runs.id, runId), + with: { taskMetrics: true }, + }) + + if (!run) { + throw new Error(`Run with ID ${runId} not found in source database`) + } + + if (!run.taskMetrics) { + throw new Error("Run is not completed") + } + + console.log(`Copying run ${run.id}`) + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { id: _, ...runTaskMetricsValues } = run.taskMetrics + const [newRunTaskMetrics] = await destDb.insert(schema.taskMetrics).values(runTaskMetricsValues).returning() + + if (!newRunTaskMetrics) { + throw new Error("Failed to insert run taskMetrics") + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { id: __, ...runValues } = run + + const [newRun] = await destDb + .insert(schema.runs) + .values({ ...runValues, taskMetricsId: newRunTaskMetrics.id }) + .returning() + + if (!newRun) { + throw new Error("Failed to insert run") + } + + const tasks = await sourceDb.query.tasks.findMany({ + where: eq(schema.tasks.runId, run.id), + with: { taskMetrics: true }, + }) + + console.log(`Copying ${tasks.length} tasks`) + + for (const task of tasks) { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { id: _, ...newTaskMetricsValues } = task.taskMetrics! + const [newTaskMetrics] = await destDb.insert(schema.taskMetrics).values(newTaskMetricsValues).returning() + + if (!newTaskMetrics) { + throw new Error(`Failed to insert taskMetrics for task ${task.id}`) + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { id: __, ...newTaskValues } = task + + const [newTask] = await destDb + .insert(schema.tasks) + .values({ ...newTaskValues, runId: newRun.id, taskMetricsId: newTaskMetrics.id }) + .returning() + + if (!newTask) { + throw new Error(`Failed to insert task ${task.id}`) + } + } + + console.log(`Successfully copied run ${runId} with ${tasks.length} tasks`) +} + +const main = async () => { + const runId = parseInt(process.argv[2], 10) + + if (isNaN(runId)) { + console.error("Run ID must be a number") + process.exit(1) + } + + try { + await copyRun(runId) + process.exit(0) + } catch (error) { + console.error(error) + process.exit(1) + } +} + +main() diff --git a/evals/turbo.json b/evals/turbo.json index 4eff691a21d..5f567ac63bc 100644 --- a/evals/turbo.json +++ b/evals/turbo.json @@ -9,7 +9,9 @@ "OPENROUTER_MODEL_ID", "PROMPT_PATH", "WORKSPACE_PATH", - "BENCHMARKS_DB_PATH" + "BENCHMARKS_DB_PATH", + "TURSO_CONNECTION_URL", + "TURSO_AUTH_TOKEN" ], "tasks": { "lint": {},