More progress

Author: cte

benchmark/package-lock.json

@@ -7,8 +7,10 @@
 		"lint": "eslint src --ext ts",
 		"check-types": "tsc --noEmit",
 		"build": "rm -rf out && tsc -p tsconfig.json",
-		"benchmark": "xvbf-run node out/run.js",
+		"benchmark": "npx dotenvx run -f .env.local -- node out/run.js",
+		"cli": "tsx src/cli.ts",
 		"clean": "rimraf out",
+		"clean:exercises": "cd exercises && git checkout -f && git clean -fd",
 		"docker:base:build": "docker build -f Dockerfile.base -t roo-code-benchmark-base ..",
 		"docker:build": "npm run docker:base:build && docker build -f Dockerfile -t roo-code-benchmark ..",
 		"docker:run": "docker run -d -it roo-code-benchmark",
@@ -22,13 +24,14 @@
 		"docker:dev:run": "docker run -d -p 2222:22 -v ~/Documents/Roo-Code:/home/vscode/repo -it roo-code-benchmark-dev",
 		"docker:dev:start": "npm run docker:dev:build && npm run docker:dev:run",
 		"docker:dev:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark-dev\" -q) /bin/bash",
-		"docker:dev:benchmark": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark-dev\" -q) npm run benchmark --",
+		"docker:dev:benchmark": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark-dev\" -q) xvbf-run npm run benchmark --",
 		"docker:dev:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-benchmark-dev\" -q)",
 		"docker:dev:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-benchmark-dev\" -q)",
 		"docker:dev:clean": "npm run docker:dev:stop && npm run docker:dev:rm"
 	},
 	"devDependencies": {
 		"@vscode/test-electron": "^2.4.0",
+		"gluegun": "^5.1.2",
 		"tsx": "^4.19.3",
 		"typescript": "^5.4.5",
 		"yargs": "^17.7.2"

benchmark/package.json

@@ -0,0 +1,9 @@
+Your job is to complete a coding exercise described by `.docs/instructions.md`.
+
+A file with the implementation stubbed out has been created for you, along with a test file.
+
+To successfully complete the exercise, you must pass all the tests in the test file.
+
+To confirm that your solution is correct, run the tests with `npm test`.
+
+Before running the tests make sure your environment is set up by running `npm install` to install the dependencies.

benchmark/prompts/javascript.md

@@ -0,0 +1,9 @@
+Your job is to complete a coding exercise described by `.docs/instructions.md`.
+
+A file with the implementation stubbed out has been created for you, along with a test file.
+
+To successfully complete the exercise, you must pass all the tests in the test file.
+
+To confirm that your solution is correct, run the tests with `yarn test`. Do not meaningfully alter the tests. You should enable the tests if they are disabled (e.g. change `xtest` to `test` for jest).
+
+Before running the tests make sure your environment is set up by running `corepack enable` to enable yarn and `yarn install` to install the dependencies.

benchmark/prompts/typescript.md

@@ -0,0 +1,91 @@
+import * as fs from "fs"
+import * as path from "path"
+import { execSync } from "child_process"
+
+import { build } from "gluegun"
+
+async function main() {
+	const cli = build()
+		.brand("roo-code-benchmark-runner")
+		.src(__dirname)
+		.help()
+		.version()
+		.defaultCommand() // Use the default command if no args.
+		.create()
+
+	const { print, prompt } = await cli.run(process.argv)
+
+	try {
+		const exercisesPath = path.resolve(__dirname, "../../benchmark/exercises")
+
+		if (!fs.existsSync(exercisesPath)) {
+			print.error(`Error: Exercises directory not found at ${exercisesPath}`)
+			process.exit(1)
+		}
+
+		const languages = fs
+			.readdirSync(exercisesPath, { withFileTypes: true })
+			.filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
+			.map((dir) => dir.name)
+
+		if (languages.length === 0) {
+			print.error("No language directories found in the exercises folder")
+			process.exit(1)
+		}
+
+		const { selectedLanguage } = await prompt.ask({
+			type: "select",
+			name: "selectedLanguage",
+			message: "Select a language:",
+			choices: languages,
+		})
+
+		const languagePath = path.join(exercisesPath, selectedLanguage)
+
+		if (!fs.existsSync(languagePath)) {
+			print.error(`Error: Language directory not found at ${languagePath}`)
+			process.exit(1)
+		}
+
+		const exercisesForLanguage = fs
+			.readdirSync(languagePath, { withFileTypes: true })
+			.filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
+			.map((dir) => dir.name)
+
+		if (exercisesForLanguage.length === 0) {
+			print.error(`No exercises found for ${selectedLanguage}`)
+			process.exit(1)
+		}
+
+		const { selectedExercise } = await prompt.ask({
+			type: "select",
+			name: "selectedExercise",
+			message: "Select an exercise:",
+			choices: exercisesForLanguage,
+		})
+
+		const exercisePath = `exercises/${selectedLanguage}/${selectedExercise}`
+		print.info(`Running ${exercisePath}...`)
+
+		try {
+			const benchmarkDir = path.resolve(__dirname, "../../benchmark")
+
+			const spinner = print.spin("Building...")
+			execSync("npm run build", { stdio: "inherit", cwd: benchmarkDir })
+			spinner.succeed("Build completed")
+
+			print.info(`Running exercise: ${exercisePath}`)
+			const runCommand = `npm run benchmark -- -e ${exercisePath}`
+			execSync(runCommand, { stdio: "inherit", cwd: benchmarkDir })
+			process.exit(0)
+		} catch (error) {
+			print.error(error)
+			process.exit(1)
+		}
+	} catch (error) {
+		print.error(error)
+		process.exit(1)
+	}
+}
+
+main()

benchmark/src/cli.ts

@@ -1,10 +1,12 @@
 import * as path from "path"
+import * as fs from "fs/promises"
 
 import yargs from "yargs"
 import { hideBin } from "yargs/helpers"
 import { runTests } from "@vscode/test-electron"
 
 async function main() {
+	// npm run build && npx dotenvx run -f .env.local -- node ./out/run.js -e exercises/javascript/binary
 	const argv = await yargs(hideBin(process.argv))
 		.option("exercise", {
 			alias: "e",
@@ -16,25 +18,15 @@ async function main() {
 
 	const extensionDevelopmentPath = path.resolve(__dirname, "../../")
 	const extensionTestsPath = path.resolve(__dirname, "./runExercise")
-	const testWorkspace = path.resolve(__dirname, argv.exercise)
+	const testWorkspace = path.resolve(__dirname, "..", argv.exercise)
+	const language = path.basename(path.dirname(testWorkspace))
+	const prompt = await fs.readFile(path.resolve(__dirname, "..", `prompts/${language}.md`), "utf-8")
 
 	await runTests({
 		extensionDevelopmentPath: extensionDevelopmentPath,
 		extensionTestsPath: extensionTestsPath,
 		launchArgs: [testWorkspace, "--disable-extensions"],
-		extensionTestsEnv: {
-			prompt: `
-				Your job is to complete a coding exercise described by \`.docs/instructions.md\`.
-				A file with the implementation stubbed out has been created for you, along with a test file.
-				To successfully complete the exercise, you must pass all the tests in the test file.
-				To confirm that your solution is correct, run the tests with \`yarn test\`.
-				Before running the test make sure your environment is set up by running \`corepack enable\` to enable yarn and \`yarn install\` to install the dependencies.
-			`
-				.trim()
-				.split("\n")
-				.map((line) => line.trim())
-				.join("\n"),
-		},
+		extensionTestsEnv: { prompt },
 	})
 }
 

benchmark/src/run.ts

@@ -4,6 +4,9 @@ import { RooCodeAPI } from "../../src/exports/roo-code"
 
 import { waitFor } from "./utils"
 
+const TIME_LIMIT = 300_000
+const STALL_LIMIT = 60_000
+
 export async function run() {
 	const extension = vscode.extensions.getExtension<RooCodeAPI>("RooVeterinaryInc.roo-cline")
 
@@ -12,49 +15,52 @@ export async function run() {
 	}
 
 	const api = extension.isActive ? extension.exports : await extension.activate()
-
-	await api.sidebarProvider.updateGlobalState("apiProvider", "openrouter")
-	await api.sidebarProvider.updateGlobalState("openRouterModelId", "anthropic/claude-3.7-sonnet")
-	await api.sidebarProvider.updateGlobalState("autoApprovalEnabled", true)
-	await api.sidebarProvider.updateGlobalState("alwaysAllowReadOnly", true)
-	await api.sidebarProvider.updateGlobalState("alwaysAllowWrite", true)
-	await api.sidebarProvider.updateGlobalState("alwaysAllowExecute", true)
-	await api.sidebarProvider.updateGlobalState("alwaysAllowBrowser", true)
-	await api.sidebarProvider.updateGlobalState("alwaysApproveResubmit", true)
-	await api.sidebarProvider.updateGlobalState("alwaysAllowMcp", true)
-	await api.sidebarProvider.updateGlobalState("alwaysAllowModeSwitch", true)
-
-	await api.sidebarProvider.storeSecret("openRouterApiKey", process.env.OPENROUTER_API_KEY!)
+	const provider = api.sidebarProvider
+	await provider.updateGlobalState("apiProvider", "openrouter")
+	await provider.updateGlobalState("openRouterModelId", "anthropic/claude-3.7-sonnet")
+	await provider.updateGlobalState("autoApprovalEnabled", true)
+	await provider.updateGlobalState("alwaysAllowReadOnly", true)
+	await provider.updateGlobalState("alwaysAllowWrite", true)
+	await provider.updateGlobalState("alwaysAllowExecute", true)
+	await provider.updateGlobalState("alwaysAllowBrowser", true)
+	await provider.updateGlobalState("alwaysApproveResubmit", true)
+	await provider.updateGlobalState("alwaysAllowMcp", true)
+	await provider.updateGlobalState("alwaysAllowModeSwitch", true)
+	await provider.storeSecret("openRouterApiKey", process.env.OPENROUTER_API_KEY!)
 
 	await vscode.workspace
 		.getConfiguration("roo-cline")
 		.update("allowedCommands", ["*"], vscode.ConfigurationTarget.Global)
 
-	await api.sidebarProvider.resolveWebviewView(
-		vscode.window.createWebviewPanel("roo-cline.SidebarProvider", "Roo Code", vscode.ViewColumn.One, {
-			enableScripts: true,
-			enableCommandUris: true,
-			retainContextWhenHidden: true,
-			localResourceRoots: [extension.extensionUri],
-		}),
-	)
-
-	await waitFor(() => api.sidebarProvider.viewLaunched)
+	await vscode.commands.executeCommand("roo-cline.SidebarProvider.focus")
+	await waitFor(() => provider.viewLaunched)
 
 	await api.startNewTask(process.env.prompt!)
 
 	let cursor = 0
 
+	const startTime = Date.now()
+
 	const getMessage = async () => {
-		await waitFor(() => api.sidebarProvider.messages.length > cursor, { timeout: 120_000 }).catch(() => {})
-		return api.sidebarProvider.messages[cursor++]
+		await waitFor(() => provider.messages.length > cursor, { timeout: STALL_LIMIT })
+		return provider.messages[cursor++]
 	}
 
 	while (true) {
-		const message = await getMessage()
-		console.log("message = ", message)
+		try {
+			const message = await getMessage()
+			console.log("message = ", message)
+
+			if (!message || message.say === "completion_result") {
+				break
+			}
+		} catch (e) {
+			console.error(e)
+			break
+		}
 
-		if (!message || message.say === "completion_result") {
+		if (Date.now() - startTime > TIME_LIMIT) {
+			console.log("Time's up!")
 			break
 		}
 	}

benchmark/src/runExercise.ts

@@ -3,16 +3,30 @@ type WaitForOptions = {
 	interval?: number
 }
 
-export const waitFor = (condition: () => boolean, { timeout = 10_000, interval = 250 }: WaitForOptions = {}) =>
-	Promise.race([
+export const waitFor = (condition: () => boolean, { timeout = 10_000, interval = 250 }: WaitForOptions = {}) => {
+	let timeoutId: NodeJS.Timeout | undefined = undefined
+
+	return Promise.race([
 		new Promise<void>((resolve) => {
-			const check = () => (condition() ? resolve() : setTimeout(check, interval))
+			const check = () => {
+				if (condition()) {
+					if (timeoutId) {
+						clearTimeout(timeoutId)
+						timeoutId = undefined
+					}
+
+					resolve()
+				} else {
+					setTimeout(check, interval)
+				}
+			}
+
 			check()
 		}),
-		new Promise((_, reject) =>
-			setTimeout(() => {
-				console.log(`Timeout after ${Math.floor(timeout / 1000)}s`)
+		new Promise((_, reject) => {
+			timeoutId = setTimeout(() => {
 				reject(new Error(`Timeout after ${Math.floor(timeout / 1000)}s`))
-			}, timeout),
-		),
+			}, timeout)
+		}),
 	])
+}

benchmark/src/utils.ts

@@ -1,3 +1,5 @@
+import * as vscode from "vscode"
+
 export interface RooCodeAPI {
 	/**
 	 * Sets the custom instructions in the global storage.