More progress

Author: cte

benchmark/.env.sample

@@ -1,3 +1,2 @@
 BENCHMARKS_DB_PATH=file:/tmp/benchmarks.db
 OPENROUTER_API_KEY=sk-or-v1-...
-POSTHOG_API_KEY=phc_...

benchmark/README.md

@@ -14,6 +14,8 @@ Install nvm:
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash
 # Reload shell.
 nvm install
+node --version
+# Verify the version is v20.18.1.
 ```
 
 Install pnpm:
@@ -29,7 +31,6 @@ Build the Roo Code extension:
 ```sh
 npm run install:all
 npx vsce package --out bin/roo-code-latest.vsix
-code --install-extension bin/roo-code-latest.vsix
 ```
 
 [Install](https://docs.docker.com/desktop/) and run Docker Desktop.
@@ -39,7 +40,9 @@ Build a container to run the Roo Code evals:
 ```sh
 cd benchmark
 pnpm install
+cp .env.sample .env
+# Update OPENROUTER_API_KEY=... with your actual API key.
 pnpm docker:start
 ```
 
-Navigation to [localhost:3000](http://localhost:3000/) in your browser.
+Navigate to [localhost:3000](http://localhost:3000/) in your browser.

benchmark/apps/cli/src/index.ts

@@ -227,9 +227,9 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server
 	})
 
 	const ignoreEvents: RooCodeEventName[] = [
-		RooCodeEventName.Message,
-		// RooCodeEventName.TaskTokenUsageUpdated,
-		// RooCodeEventName.TaskAskResponded,
+		// RooCodeEventName.Message,
+		RooCodeEventName.TaskTokenUsageUpdated,
+		RooCodeEventName.TaskAskResponded,
 	]
 
 	let taskStartedAt = Date.now()
@@ -248,7 +248,7 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server
 
 		if (!ignoreEvents.includes(eventName)) {
 			console.log(`[cli#runExercise | ${language} / ${exercise}] taskEvent -> ${eventName}`)
-			// console.log(payload)
+			console.log(payload)
 		}
 
 		if (eventName === RooCodeEventName.TaskStarted) {

benchmark/package.json

@@ -12,11 +12,11 @@
 		"cli": "turbo dev --filter @benchmark/cli --output-logs new-only --ui tui -- run",
 		"drizzle:studio": "pnpm --filter @benchmark/db db:studio",
 		"docker:build": "docker build -f Dockerfile -t roo-code-benchmark --progress=plain ..",
-		"docker:run": "touch /tmp/benchmarks.db && docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark",
+		"docker:run": "docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark",
 		"docker:start": "pnpm docker:build && pnpm docker:run",
-		"docker:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) /bin/bash",
-		"docker:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-benchmark\" -q)",
-		"docker:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-benchmark\" -q)",
+		"docker:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) /bin/zsh",
+		"docker:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) >/dev/null 2>&1 || true",
+		"docker:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-benchmark\" -q) >/dev/null 2>&1 || true",
 		"docker:clean": "pnpm docker:stop && pnpm docker:rm"
 	},
 	"devDependencies": {