Improve Docker setup for evals

.dockerignore

@@ -1,18 +1,31 @@
-# Build artifacts
+# git
+.git
+
+# build artifacts
 bin/
-!bin/roo-code-latest.vsix
 dist/
 **/dist/
 out/
 **/out/
+src/webview-ui/
 
-# Dependencies
+# dependencies
 node_modules/
 **/node_modules/
 
-# Test and development files
+# testing
 coverage/
 **/.vscode-test/
+**/mock/
 
+# devtools
 knip.json
 .husky/
+
+# monorepo
+.turbo/
+**/.turbo/
+
+# next.js
+**/.next/
+.vercel

apps/web-evals/src/lib/server/runs.ts

@@ -1,3 +1,5 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+
 "use server"
 
 import { spawn } from "child_process"
@@ -49,24 +51,60 @@ export async function createRun({ suite, exercises = [], systemPrompt, ...values
 
 	revalidatePath("/runs")
 
-	try {
-		const logFile = fs.openSync(`/tmp/roo-code-evals-${run.id}.log`, "a")
+	// try {
+	// 	const logFile = fs.openSync(`/tmp/roo-code-evals-${run.id}.log`, "a")
 
-		const env: NodeJS.ProcessEnv = systemPrompt
-			? { ...process.env, FOOTGUN_SYSTEM_PROMPT: systemPrompt }
-			: process.env
+	// 	const env: NodeJS.ProcessEnv = systemPrompt
+	// 		? { ...process.env, FOOTGUN_SYSTEM_PROMPT: systemPrompt }
+	// 		: process.env
 
-		const childProcess = spawn("pnpm", ["--filter", "@roo-code/evals", "cli", run.id.toString()], {
-			detached: true,
-			stdio: ["ignore", logFile, logFile],
-			env,
-		})
+	// 	const childProcess = spawn("pnpm", ["--filter", "@roo-code/evals", "cli", run.id.toString()], {
+	// 		detached: true,
+	// 		stdio: ["ignore", logFile, logFile],
+	// 		env,
+	// 	})
 
-		childProcess.unref()
-		await _updateRun(run.id, { pid: childProcess.pid })
-	} catch (error) {
-		console.error(error)
-	}
+	// 	childProcess.unref()
+	// 	await _updateRun(run.id, { pid: childProcess.pid })
+	// } catch (error) {
+	// 	console.error(error)
+	// }
+
+	// try {
+	// 	const logFile = `/tmp/roo-code-evals-${run.id}.log`
+
+	// 	const envVars = systemPrompt ? { ...process.env, FOOTGUN_SYSTEM_PROMPT: systemPrompt } : process.env
+
+	// 	// Requires a docker socket mounted and host container running.
+	// 	const runOnHost = async () => {
+	// 		// Create and start a new runner container connected to the compose network
+	// 		const command = `docker run --rm --network evals_default evals-runner sh -c "pnpm --filter @roo-code/evals cli ${run.id}"`
+
+	// 		const childProcess = spawn("sh", ["-c", command], {
+	// 			detached: true,
+	// 			stdio: ["ignore", "pipe", "pipe"],
+	// 		})
+
+	// 		// Redirect output to log file
+	// 		const logStream = fs.createWriteStream(logFile, { flags: "a" })
+
+	// 		if (childProcess.stdout) {
+	// 			childProcess.stdout.pipe(logStream)
+	// 		}
+
+	// 		if (childProcess.stderr) {
+	// 			childProcess.stderr.pipe(logStream)
+	// 		}
+
+	// 		return childProcess
+	// 	}
+
+	// 	const childProcess = await runOnHost()
+	// 	childProcess.unref()
+	// 	await _updateRun(run.id, { pid: childProcess.pid })
+	// } catch (error) {
+	// 	console.error(error)
+	// }
 
 	return run
 }

packages/evals/Dockerfile.runner

@@ -0,0 +1,101 @@
+# docker build -f packages/evals/Dockerfile.runner -t evals-runner .
+# docker run -it evals-runner
+
+FROM node:20-slim AS base
+
+# Install pnpm
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+RUN npm install -g npm@latest
+RUN npm install -g npm-run-all
+
+# Install system packages
+RUN apt update && apt install -y sudo curl git vim jq
+
+# Create a `vscode` user
+RUN useradd -m vscode -s /bin/bash && \
+  echo "vscode ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/vscode && \
+  chmod 0440 /etc/sudoers.d/vscode
+
+# Install VS Code
+# https://code.visualstudio.com/docs/setup/linux
+RUN apt install -y wget gpg apt-transport-https
+RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg
+RUN install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg
+RUN echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" | tee /etc/apt/sources.list.d/vscode.list > /dev/null
+RUN rm -f packages.microsoft.gpg
+RUN apt update && apt install -y code
+
+# Install Xvfb
+RUN apt install -y xvfb
+
+# [cpp] Install cmake 3.28.3
+RUN apt install -y cmake
+
+# [go] Install Go 1.22.2
+RUN apt install -y golang-go
+
+# [java] Install Java 21
+RUN apt install -y default-jre
+
+# [python] Install Python 3.12.3 and uv 0.6.6
+RUN apt install -y python3 python3-venv python3-dev python3-pip
+
+WORKDIR /home/vscode
+USER vscode
+
+# [rust] Install Rust 1.85
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc
+
+# Copy evals
+RUN git clone https://github.com/RooCodeInc/Roo-Code-Evals.git evals
+
+# Prepare evals
+WORKDIR /home/vscode/evals/python
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+RUN /home/vscode/.local/bin/uv sync
+
+WORKDIR /home/vscode/repo
+
+# Install npm packages
+RUN mkdir -p \
+  scripts \
+  packages/build \
+  packages/cloud \
+  packages/config-eslint \
+  packages/config-typescript \
+  packages/evals \
+  packages/ipc \
+  packages/telemetry \
+  packages/types \
+  src \
+  webview-ui
+
+COPY --chown=vscode:vscode ./package.json                            ./
+COPY --chown=vscode:vscode ./pnpm-lock.yaml                          ./
+COPY --chown=vscode:vscode ./pnpm-workspace.yaml                     ./
+COPY --chown=vscode:vscode ./scripts/bootstrap.mjs                   ./scripts/
+COPY --chown=vscode:vscode ./packages/build/package.json             ./packages/build/
+COPY --chown=vscode:vscode ./packages/cloud/package.json             ./packages/cloud/
+COPY --chown=vscode:vscode ./packages/config-eslint/package.json     ./packages/config-eslint/
+COPY --chown=vscode:vscode ./packages/config-typescript/package.json ./packages/config-typescript/
+COPY --chown=vscode:vscode ./packages/evals/package.json             ./packages/evals/
+COPY --chown=vscode:vscode ./packages/ipc/package.json               ./packages/ipc/
+COPY --chown=vscode:vscode ./packages/telemetry/package.json         ./packages/telemetry/
+COPY --chown=vscode:vscode ./packages/types/package.json             ./packages/types/
+COPY --chown=vscode:vscode ./src/package.json                        ./src/
+COPY --chown=vscode:vscode ./webview-ui/package.json                 ./webview-ui/
+RUN pnpm install
+
+# Build the extension
+COPY --chown=vscode:vscode . ./
+RUN pnpm build -- --out ../bin/roo-code.vsix
+
+# Copy & install extension
+RUN code --debug --install-extension bin/roo-code.vsix
+
+ENV DATABASE_URL=postgresql://postgres:password@db:5432/evals_development
+
+CMD ["bash"]

packages/evals/Dockerfile.web

@@ -0,0 +1,72 @@
+# docker build -f packages/evals/Dockerfile.web -t evals-web .
+# docker run -it evals-web
+
+FROM node:20-slim AS base
+
+# Install pnpm
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+RUN npm install -g npm@latest
+RUN npm install -g npm-run-all
+
+# Install system packages
+RUN apt update && apt install -y sudo curl git vim jq
+
+# Install Docker CLI
+RUN apt install -y apt-transport-https ca-certificates gnupg lsb-release
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+RUN apt update && apt install -y docker-ce-cli
+
+# Create a `vscode` user and add to docker group
+RUN useradd -m vscode -s /bin/bash && \
+  echo "vscode ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/vscode && \
+  chmod 0440 /etc/sudoers.d/vscode
+
+WORKDIR /home/vscode
+USER vscode
+
+# Copy evals
+RUN git clone https://github.com/RooCodeInc/Roo-Code-Evals.git evals
+
+WORKDIR /home/vscode/repo
+
+# Install npm packages
+RUN mkdir -p \
+  scripts \
+  apps/web-evals \
+  packages/build \
+  packages/cloud \
+  packages/config-eslint \
+  packages/config-typescript \
+  packages/evals \
+  packages/ipc \
+  packages/telemetry \
+  packages/types \
+  src \
+  webview-ui
+
+COPY --chown=vscode:vscode ./package.json                            ./
+COPY --chown=vscode:vscode ./pnpm-lock.yaml                          ./
+COPY --chown=vscode:vscode ./pnpm-workspace.yaml                     ./
+COPY --chown=vscode:vscode ./scripts/bootstrap.mjs                   ./scripts/
+COPY --chown=vscode:vscode ./apps/web-evals/package.json             ./apps/web-evals/
+COPY --chown=vscode:vscode ./packages/build/package.json             ./packages/build/
+COPY --chown=vscode:vscode ./packages/cloud/package.json             ./packages/cloud/
+COPY --chown=vscode:vscode ./packages/config-eslint/package.json     ./packages/config-eslint/
+COPY --chown=vscode:vscode ./packages/config-typescript/package.json ./packages/config-typescript/
+COPY --chown=vscode:vscode ./packages/evals/package.json             ./packages/evals/
+COPY --chown=vscode:vscode ./packages/ipc/package.json               ./packages/ipc/
+COPY --chown=vscode:vscode ./packages/telemetry/package.json         ./packages/telemetry/
+COPY --chown=vscode:vscode ./packages/types/package.json             ./packages/types/
+COPY --chown=vscode:vscode ./src/package.json                        ./src/
+COPY --chown=vscode:vscode ./webview-ui/package.json                 ./webview-ui/
+RUN pnpm install
+
+COPY --chown=vscode:vscode . ./
+
+ENV DATABASE_URL=postgresql://postgres:password@db:5432/evals_development
+
+EXPOSE 3000
+CMD ["pnpm", "--filter", "@roo-code/web-evals", "dev"]