Dockerize evals

.dockerignore

@@ -7,6 +7,7 @@
 
 # Build artifacts
 bin/
+!bin/roo-code-latest.vsix
 dist/
 **/dist/
 out/

benchmark/Dockerfile

@@ -1,8 +1,3 @@
-# docker build -f Dockerfile.base -t roo-code-benchmark-base ..
-# docker build -f Dockerfile -t roo-code-benchmark ..
-# docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark
-# docker exec -it $(docker ps --filter "ancestor=roo-code-benchmark" -q) /bin/bash
-
 FROM node:20-slim AS base
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
@@ -49,34 +44,43 @@ RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc
 WORKDIR /home/vscode
 USER vscode
 
-COPY benchmark/entrypoint.sh /usr/local/bin/entrypoint.sh
-
 # Copy exercises
-WORKDIR /home/vscode
 RUN git clone https://github.com/cte/Roo-Code-Benchmark.git exercises
 
 # Prepare exercises
 WORKDIR /home/vscode/exercises/python
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 RUN /home/vscode/.local/bin/uv sync
 
-# Copy and build extension dependencies
-WORKDIR /home/vscode/repo
-COPY --chown=vscode:vscode package*.json /home/vscode/repo/
-COPY --chown=vscode:vscode webview-ui/package*.json /home/vscode/repo/webview-ui/
-RUN npm-run-all -l -p install-extension install-webview
+WORKDIR /home/vscode/repo/benchmark
 
-# Copy extension and benchmark code
-COPY --chown=vscode:vscode . /home/vscode/repo
+# Copy & install extension
+COPY --chown=vscode:vscode ./bin/roo-code-latest.vsix ./
+RUN code --debug --install-extension ./roo-code-latest.vsix
 
-# Build extension dependencies
-WORKDIR /home/vscode/repo/benchmark
+# Install dependencies
+COPY --chown=vscode:vscode ./benchmark/package.json ./benchmark/pnpm-lock.yaml ./benchmark/pnpm-workspace.yaml ./benchmark/.npmrc ./
+RUN mkdir -p apps/cli apps/web \
+  config/eslint config/typescript \
+  packages/db packages/ipc packages/lib packages/types
+COPY --chown=vscode:vscode ./benchmark/apps/cli/package.json          ./apps/cli/
+COPY --chown=vscode:vscode ./benchmark/apps/web/package.json          ./apps/web/
+COPY --chown=vscode:vscode ./benchmark/config/eslint/package.json     ./config/eslint/
+COPY --chown=vscode:vscode ./benchmark/config/typescript/package.json ./config/typescript/
+COPY --chown=vscode:vscode ./benchmark/packages/db/package.json       ./packages/db/
+COPY --chown=vscode:vscode ./benchmark/packages/ipc/package.json      ./packages/ipc/
+COPY --chown=vscode:vscode ./benchmark/packages/lib/package.json      ./packages/lib/
+COPY --chown=vscode:vscode ./benchmark/packages/types/package.json    ./packages/types/
 RUN pnpm install
 
-# Initialize database
-RUN echo "BENCHMARKS_DB_PATH=file:/tmp/benchmarks.db" > .env
+# Copy application code
+COPY --chown=vscode:vscode ./benchmark ./
+
+# Copy environment variables
+COPY --chown=vscode:vscode ./benchmark/.env ./
+
+# Push database schema
 RUN pnpm --filter @benchmark/db db:push
 
 EXPOSE 3000
-ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
-CMD ["/usr/bin/pnpm", "dev"]
+CMD ["pnpm", "web"]

benchmark/README.md

@@ -2,16 +2,12 @@
 
 ## Get Started
 
-[Install](https://docs.docker.com/desktop/) and run Docker Desktop.
-
-Build a container to run the Roo Code benchmarks:
+Clone the Roo Code repo:
 
 ```sh
-docker build -f Dockerfile -t roo-code-benchmark ..
+git clone https://github.com/RooVetGit/Roo-Code.git
 ```
 
-## Local Debugging
-
 Install nvm:
 
 ```sh
@@ -28,28 +24,21 @@ corepack enable pnpm
 corepack use pnpm@latest-10
 ```
 
-Install dependencies:
+Build the Roo Code extension:
 
 ```sh
-pnpm install
+npm run install:all
+npx vsce package --out bin/roo-code-latest.vsix
 ```
 
-Configure database:
-
-```sh
-cp packages/server/.env.sample packages/server/.env
-# Update BENCHMARKS_DB_PATH as needed in `packages/server/.env`.
-pnpm --filter @benchmark/db db:push
-```
+[Install](https://docs.docker.com/desktop/) and run Docker Desktop.
 
-Run the web app:
+Build a container to run the Roo Code evals:
 
 ```sh
-pnpm web
+cd benchmark
+pnpm install
+pnpm docker:start
 ```
 
-Run an exercise via the cli:
-
-```sh
-pnpm cli run [cpp|go|java|javascript|python|rust|all] [<exercise>|all]
-```
+Navigation to [localhost:3000](http://localhost:3000/) in your browser.

benchmark/apps/cli/src/index.ts

@@ -147,10 +147,20 @@ const run = async (toolbox: GluegunToolbox) => {
 	await Promise.all(runningPromises)
 
 	const result = await finishRun(run.id)
-	console.log("[cli#run]", result)
+	try {
+		console.log("[cli#run]", result)
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	} catch (error) {
+		// console.error(error)
+	}
 
 	if (parentPid) {
-		console.log(await execa`kill -INT ${parentPid}`)
+		try {
+			console.log(await execa`kill -INT ${parentPid}`)
+			// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		} catch (error) {
+			// console.error(error)
+		}
 	}
 
 	console.log(await execa({ cwd: exercisesPath })`git add .`)
@@ -163,11 +173,24 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server
 	const dirname = path.dirname(run.socketPath)
 	const taskSocketPath = path.resolve(dirname, `${dirname}/task-${task.id}.sock`)
 
-	await execa({
-		env: { ROO_CODE_IPC_SOCKET_PATH: taskSocketPath },
-	})`code -n ${path.resolve(exercisesPath, language, exercise)}`
+	let codeCommand = `code --wait --verbose --disable-workspace-trust`
+
+	const isDocker = fs.existsSync("/.dockerenv")
+
+	if (isDocker) {
+		codeCommand = `xvfb-run --auto-servernum --server-num=1 ${codeCommand} --disable-gpu --password-store="basic"`
+	}
+
+	const subprocess = execa({
+		env: {
+			ROO_CODE_IPC_SOCKET_PATH: taskSocketPath,
+		},
+		shell: "/bin/bash",
+	})`${codeCommand} -n ${path.resolve(exercisesPath, language, exercise)}`
+
+	subprocess.stdout.pipe(process.stdout)
 
-	console.log(`Connecting to ${taskSocketPath}`)
+	console.log(`Connecting to ${taskSocketPath} (pid: ${subprocess.pid})`)
 
 	const createClient = (taskSocketPath: string) => {
 		const ipcClient = new IpcClient(taskSocketPath)
@@ -208,10 +231,10 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server
 		isTaskFinished = true
 	})
 
-	const ignoreEvents = [
-		RooCodeEventName.Message,
-		RooCodeEventName.TaskTokenUsageUpdated,
-		RooCodeEventName.TaskAskResponded,
+	const ignoreEvents: RooCodeEventName[] = [
+		// RooCodeEventName.Message,
+		// RooCodeEventName.TaskTokenUsageUpdated,
+		// RooCodeEventName.TaskAskResponded,
 	]
 
 	let taskStartedAt = Date.now()
@@ -230,6 +253,7 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server
 
 		if (!ignoreEvents.includes(eventName)) {
 			console.log(`[cli#runExercise | ${language} / ${exercise}] taskEvent -> ${eventName}`)
+			console.log(payload)
 		}
 
 		if (eventName === RooCodeEventName.TaskStarted) {

benchmark/package.json

@@ -11,14 +11,13 @@
 		"web": "turbo dev --filter @benchmark/web --output-logs new-only --ui tui",
 		"cli": "turbo dev --filter @benchmark/cli --output-logs new-only --ui tui -- run",
 		"drizzle:studio": "pnpm --filter @benchmark/db db:studio",
-		"docker:build": "docker build -f Dockerfile -t roo-code-benchmark ..",
+		"docker:build": "docker build -f Dockerfile -t roo-code-benchmark --progress=plain ..",
 		"docker:run": "touch /tmp/benchmarks.db && docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark",
-		"docker:start": "npm run docker:build && npm run docker:run",
+		"docker:start": "pnpm docker:build && pnpm docker:run",
 		"docker:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) /bin/bash",
-		"docker:cli": "docker exec -it -w /home/vscode/repo/benchmark $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) xvfb-run npm run cli --",
 		"docker:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-benchmark\" -q)",
 		"docker:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-benchmark\" -q)",
-		"docker:clean": "npm run docker:stop && npm run docker:rm"
+		"docker:clean": "pnpm docker:stop && pnpm docker:rm"
 	},
 	"devDependencies": {
 		"@dotenvx/dotenvx": "^1.39.0",

src/exports/api.ts

@@ -4,7 +4,7 @@ import * as vscode from "vscode"
 import { ClineProvider } from "../core/webview/ClineProvider"
 import { openClineInNewTab } from "../activate/registerCommands"
 
-import { RooCodeSettings, RooCodeEvents, RooCodeEventName } from "../schemas"
+import { RooCodeSettings, RooCodeEvents, RooCodeEventName, ClineMessage } from "../schemas"
 import { IpcOrigin, IpcMessageType, TaskCommandName, TaskEvent } from "../schemas/ipc"
 import { RooCodeAPI } from "./interface"
 import { IpcServer } from "./ipc"
@@ -28,28 +28,75 @@ export class API extends EventEmitter<RooCodeEvents> implements RooCodeAPI {
 		this.registerListeners(this.sidebarProvider)
 
 		if (socketPath) {
-			this.ipc = new IpcServer(socketPath, (...args: unknown[]) => outputChannelLog(this.outputChannel, ...args))
-			this.ipc.listen()
+			const ipc = (this.ipc = new IpcServer(socketPath, (...args: unknown[]) =>
+				outputChannelLog(this.outputChannel, ...args),
+			))
 
-			this.outputChannel.appendLine(
-				`[API] ipc server started: socketPath=${socketPath}, pid=${process.pid}, ppid=${process.ppid}`,
-			)
-
-			this.ipc.on(IpcMessageType.TaskCommand, async (_clientId, { commandName, data }) => {
-				this.outputChannel.appendLine(`[API] TaskCommand -> ${commandName}`)
+			ipc.listen()
+			this.log(`[API] ipc server started: socketPath=${socketPath}, pid=${process.pid}, ppid=${process.ppid}`)
 
+			ipc.on(IpcMessageType.TaskCommand, async (_clientId, { commandName, data }) => {
 				switch (commandName) {
 					case TaskCommandName.StartNewTask:
-						await this.startNewTask(data)
+						this.log(`[API] StartNewTask -> ${data.text}`)
+						this.log(`[API] StartNewTask -> ${JSON.stringify(data.configuration)}`)
+
+						try {
+							await this.startNewTask(data)
+
+							ipc.broadcast({
+								type: IpcMessageType.TaskEvent,
+								origin: IpcOrigin.Server,
+								data: {
+									eventName: RooCodeEventName.Message,
+									payload: [
+										{
+											taskId: "[system]",
+											action: "created",
+											message: {
+												ts: Date.now(),
+												type: "say",
+												text: `ACK: TaskCommand -> ${commandName}`,
+											},
+										},
+									],
+								},
+							})
+						} catch (error) {
+							this.log(`[API] error starting new task: ${error}`)
+						}
+
 						break
 					case TaskCommandName.CancelTask:
+						this.log(`[API] CancelTask -> ${data}`)
+
 						await this.cancelTask(data)
+
+						ipc.broadcast({
+							type: IpcMessageType.TaskEvent,
+							origin: IpcOrigin.Server,
+							data: {
+								eventName: RooCodeEventName.Message,
+								payload: [
+									{
+										taskId: "[system]",
+										action: "created",
+										message: {
+											ts: Date.now(),
+											type: "say",
+											text: `ACK: CancelTask -> ${data}`,
+										},
+									},
+								],
+							},
+						})
+
 						break
 				}
 			})
 
-			this.ipc.on(IpcMessageType.VSCodeCommand, async (_clientId, command) => {
-				this.outputChannel.appendLine(`[API] VSCodeCommand -> ${command}`)
+			ipc.on(IpcMessageType.VSCodeCommand, async (_clientId, command) => {
+				this.log(`[API] VSCodeCommand -> ${command}`)
 				await vscode.commands.executeCommand(command)
 			})
 		}
@@ -78,11 +125,13 @@ export class API extends EventEmitter<RooCodeEvents> implements RooCodeAPI {
 		let provider: ClineProvider
 
 		if (newTab) {
-			await vscode.commands.executeCommand("workbench.action.closeAllEditors")
+			// await vscode.commands.executeCommand("workbench.action.closeAllEditors")
 
 			if (!this.tabProvider) {
+				this.log(`[API#startNewTask] -> openClineInNewTab`)
 				this.tabProvider = await openClineInNewTab({ context: this.context, outputChannel: this.outputChannel })
 				this.registerListeners(this.tabProvider)
+				this.log(`[API#startNewTask] <- openClineInNewTab`)
 			}
 
 			provider = this.tabProvider
@@ -91,21 +140,32 @@ export class API extends EventEmitter<RooCodeEvents> implements RooCodeAPI {
 		}
 
 		if (configuration) {
+			this.log(`[API#startNewTask] -> setValues ${JSON.stringify(configuration)}`)
 			await provider.setValues(configuration)
+			this.log(`[API#startNewTask] <- setValues ${JSON.stringify(configuration)}`)
 
 			if (configuration.allowedCommands) {
+				this.log(`[API#startNewTask] -> setAllowedCommands ${JSON.stringify(configuration.allowedCommands)}`)
+
 				await vscode.workspace
 					.getConfiguration("roo-cline")
 					.update("allowedCommands", configuration.allowedCommands, vscode.ConfigurationTarget.Global)
+
+				this.log(`[API#startNewTask] setAllowedCommands ${JSON.stringify(configuration.allowedCommands)}`)
 			}
 		}
 
+		this.log(`[API#startNewTask] -> removeClineFromStack`)
 		await provider.removeClineFromStack()
 		await provider.postStateToWebview()
 		await provider.postMessageToWebview({ type: "action", action: "chatButtonClicked" })
 		await provider.postMessageToWebview({ type: "invoke", invoke: "newChat", text, images })
+		this.log(`[API#startNewTask] <- removeClineFromStack`)
 
+		this.log(`[API#startNewTask] -> initClineWithTask`)
 		const { taskId } = await provider.initClineWithTask(text, images)
+		this.log(`[API#startNewTask] <- initClineWithTask`)
+
 		return taskId
 	}
 
@@ -164,6 +224,7 @@ export class API extends EventEmitter<RooCodeEvents> implements RooCodeAPI {
 
 	public log(message: string) {
 		this.outputChannel.appendLine(message)
+		console.log(`${message}\n`)
 	}
 
 	private registerListeners(provider: ClineProvider) {