Skip to content

Commit 283e201

Browse files
ctecte
committed
Evals
1 parent edb53bf commit 283e201

File tree

19 files changed

+1819
-32
lines changed

19 files changed

+1819
-32
lines changed

‎.dockerignore‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Version control
2+
# .git/
3+
# .gitignore
4+
# .gitattributes
5+
# .git-blame-ignore-revs
6+
# .gitconfig
7+
8+
# Build artifacts
9+
bin/
10+
dist/
11+
**/dist/
12+
out/
13+
**/out/
14+
15+
# Dependencies
16+
node_modules/
17+
**/node_modules/
18+
19+
# Test and development files
20+
coverage/
21+
**/.vscode-test/
22+
23+
# Configuration files
24+
# .env*
25+
knip.json
26+
.husky/
27+
28+
# CI/CD
29+
# .changeset/
30+
# .github/
31+
# ellipsis.yaml
32+
33+
# OS specific
34+
.DS_Store
35+
36+
# Logs
37+
logs/
38+
*.log
39+
40+
# Nix
41+
# flake.lock
42+
# flake.nix
43+
44+
# Monorepo
45+
benchmark/exercises/

‎.vscodeignore‎

Lines changed: 31 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,62 @@
11
# Default
2+
.changeset/**
23
.github/**
34
.husky/**
45
.vscode/**
5-
.vscode-test/**
6-
out/**
7-
out-integration/**
8-
e2e/**
6+
coverage/**
97
node_modules/**
108
src/**
11-
.gitignore
12-
.yarnrc
13-
esbuild.js
14-
vsc-extension-quickstart.md
9+
**/.npmrc
1510
**/tsconfig.json
1611
**/.eslintrc.json
1712
**/*.map
1813
**/*.ts
1914
**/.vscode-test.*
15+
**/.gitignore
16+
**/.DS_Store
2017

2118
# Custom
22-
demo.gif
23-
.nvmrc
19+
ellipsis.yaml
20+
esbuild.js
21+
flake.*
22+
jest.config.js
23+
knip.json
24+
assets/docs/demo.gif
25+
.clinerules*
26+
.dockerignore
27+
.git-blame-ignore-revs
2428
.gitattributes
29+
.gitconfig
30+
.nvmrc
2531
.prettierignore
26-
.clinerules*
32+
.rooignore
2733
.roomodes
34+
benchmark/**
2835
cline_docs/**
29-
coverage/**
36+
e2e/**
37+
out/**
3038

31-
# Ignore all webview-ui files except the build directory (https://github.com/microsoft/vscode-webview-ui-toolkit-samples/blob/main/frameworks/hello-world-react-cra/.vscodeignore)
32-
webview-ui/src/**
39+
# Ignore all webview-ui files except the build directory.
40+
# https://github.com/microsoft/vscode-webview-ui-toolkit-samples/blob/main/frameworks/hello-world-react-cra/.vscodeignore
41+
webview-ui/.storybook/**
3342
webview-ui/public/**
34-
webview-ui/scripts/**
43+
webview-ui/src/**
3544
webview-ui/index.html
36-
webview-ui/README.md
37-
webview-ui/package.json
45+
webview-ui/jest.config.cjs
3846
webview-ui/package-lock.json
47+
webview-ui/package.json
3948
webview-ui/node_modules/**
40-
**/.gitignore
4149

42-
# Fix issue where codicons don't get packaged (https://github.com/microsoft/vscode-extension-samples/issues/692)
50+
# Fix issue where codicons don't get packaged.
51+
# https://github.com/microsoft/vscode-extension-samples/issues/692
4352
!node_modules/@vscode/codicons/dist/codicon.css
4453
!node_modules/@vscode/codicons/dist/codicon.ttf
4554

46-
# Include default themes JSON files used in getTheme
55+
# Include default themes JSON files used in getTheme.
4756
!src/integrations/theme/default-themes/**
4857

49-
# Include icons
58+
# Include icons.
5059
!assets/icons/**
5160

52-
# Include .env file for telemetry
61+
# Include .env file for telemetry.
5362
!.env

‎benchmark/.env.local.sample‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
OPENROUTER_API_KEY=sk-or-v1-...
2+
POSTHOG_API_KEY=phc_...

‎benchmark/.gitignore‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
exercises
2+
3+
.pnp.*
4+
5+
.yarn
6+
7+
.env.*
8+
!.env.*.sample

‎benchmark/Dockerfile‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# docker build -f Dockerfile -t roo-code-benchmark ..
2+
# docker run -d -it roo-code-benchmark
3+
# docker exec -it $(docker ps --filter "ancestor=roo-code-benchmark" -q) /bin/bash
4+
5+
FROM ubuntu:latest
6+
7+
# Install dependencies
8+
RUN apt update && apt install -y curl git vim
9+
10+
# Install VS Code
11+
# https://code.visualstudio.com/docs/setup/linux
12+
RUN apt update && apt install -y wget gpg apt-transport-https
13+
RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg
14+
RUN install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg
15+
RUN echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" | tee /etc/apt/sources.list.d/vscode.list > /dev/null
16+
RUN rm -f packages.microsoft.gpg
17+
RUN apt update && apt install -y code
18+
19+
# Install Xvfb
20+
RUN apt install -y xvfb
21+
22+
# Install node.js
23+
RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -
24+
RUN apt update && apt install -y nodejs
25+
26+
# Create a `vscode` user
27+
RUN apt install -y sudo && \
28+
useradd -m vscode -s /bin/bash && \
29+
echo "vscode ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/vscode && \
30+
chmod 0440 /etc/sudoers.d/vscode
31+
32+
WORKDIR /home/vscode
33+
USER vscode
34+
35+
COPY benchmark/entrypoint.sh /usr/local/bin/
36+
COPY --chown=vscode:vscode . /home/vscode/repo
37+
38+
WORKDIR /home/vscode/repo
39+
RUN npm run install-extension
40+
RUN npm run compile
41+
42+
WORKDIR /home/vscode/repo/webview-ui
43+
RUN npm install
44+
RUN npm run build
45+
46+
WORKDIR /home/vscode/repo/benchmark
47+
RUN npm install
48+
RUN npm run build
49+
RUN git clone https://github.com/cte/Roo-Code-Benchmark.git exercises
50+
51+
CMD ["/bin/bash"]
52+
53+
# FROM buildpack-deps:jammy
54+
55+
# # Install Python 3.11
56+
# RUN apt-get update && apt-get install -y \
57+
# software-properties-common \
58+
# cmake \
59+
# libboost-all-dev \
60+
# && add-apt-repository ppa:deadsnakes/ppa \
61+
# && apt-get update \
62+
# && apt-get install -y \
63+
# python3.11 \
64+
# python3.11-venv \
65+
# python3.11-dev \
66+
# python3-pip \
67+
# ca-certificates-java \
68+
# openjdk-21-jdk \
69+
# libtbb-dev \
70+
# && rm -rf /var/lib/apt/lists/*
71+
72+
# # Make python3.11 the default python3
73+
# RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
74+
75+
# # Install Go with architecture detection
76+
# RUN ARCH=$(uname -m) && \
77+
# if [ "$ARCH" = "x86_64" ]; then \
78+
# GOARCH="amd64"; \
79+
# elif [ "$ARCH" = "aarch64" ]; then \
80+
# GOARCH="arm64"; \
81+
# else \
82+
# false; \
83+
# fi && \
84+
# curl -L "https://golang.org/dl/go1.21.5.linux-$GOARCH.tar.gz" -o go.tar.gz && \
85+
# tar -C /usr/local -xzf go.tar.gz && \
86+
# rm go.tar.gz
87+
# ENV PATH="/usr/local/go/bin:${PATH}"
88+
89+
# # Install Rust
90+
# ADD https://sh.rustup.rs /tmp/rustup.sh
91+
# RUN chmod +x /tmp/rustup.sh && /tmp/rustup.sh -y && rm /tmp/rustup.sh
92+
# ENV PATH="/root/.cargo/bin:${PATH}"
93+
94+
# # Install Node.js and dependencies
95+
# RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
96+
# apt-get install -y nodejs && \
97+
# rm -rf /var/lib/apt/lists/* && \
98+
# mkdir -p /npm-install && \
99+
# cd /npm-install && \
100+
# npm init -y && \
101+
# npm install \
102+
# jest \
103+
# @babel/[email protected] \
104+
# @exercism/[email protected] \
105+
# @exercism/[email protected] \
106+
# @types/[email protected] \
107+
# @types/[email protected] \
108+
# [email protected] \
109+
# [email protected] \
110+
# [email protected]
111+
112+
# COPY . /aider
113+
# RUN pip3 install --no-cache-dir --upgrade pip uv
114+
# RUN uv pip install --system --no-cache-dir -e /aider[dev]
115+
# RUN git config --global --add safe.directory /aider
116+
# WORKDIR /aider

‎benchmark/Dockerfile.dev‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# docker build -f Dockerfile.dev -t roo-code-benchmark-dev ..
2+
# docker run -d -p 2222:22 -it roo-code-benchmark-dev
3+
# docker exec -it $(docker ps --filter "ancestor=roo-code-benchmark-dev" -q) /bin/bash
4+
# ssh vscode@localhost -p 2222
5+
# Or, use VSCode SSH remote extension to connect.
6+
7+
FROM ubuntu:latest
8+
9+
# Install dependencies
10+
RUN apt update && apt install -y curl git vim
11+
12+
# Install VS Code
13+
# https://code.visualstudio.com/docs/setup/linux
14+
RUN apt update && apt install -y wget gpg apt-transport-https
15+
RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg
16+
RUN install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg
17+
RUN echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" | tee /etc/apt/sources.list.d/vscode.list > /dev/null
18+
RUN rm -f packages.microsoft.gpg
19+
RUN apt update && apt install -y code
20+
21+
# Install Xvfb
22+
RUN apt install -y xvfb
23+
24+
# Install SSH server
25+
RUN apt install -y openssh-server
26+
RUN mkdir -p /var/run/sshd && \
27+
echo 'PermitRootLogin no' >> /etc/ssh/sshd_config && \
28+
echo 'PasswordAuthentication yes' >> /etc/ssh/sshd_config
29+
RUN ssh-keygen -A
30+
31+
# Install node.js
32+
RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -
33+
RUN apt update && apt install -y nodejs
34+
35+
# Create a `vscode` user
36+
RUN apt install -y sudo && \
37+
useradd -m vscode -s /bin/bash && \
38+
echo "vscode ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/vscode && \
39+
chmod 0440 /etc/sudoers.d/vscode
40+
41+
WORKDIR /home/vscode
42+
USER vscode
43+
44+
RUN mkdir -p /home/vscode/.ssh
45+
RUN echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKGzg3BQ0QHU4m6G6CAdQ57LnBVljlPWfdySrsjV3twQaOYoJ8eEy1ck7kYZfH7DIbCteH4hkNIk32ghjWE84j3unO2/3wGC+CId9raZMudi8UbNgAFMYZZqTcrWR1lCxSLpDNT01JHYHw4BCMkJ4XjH2so+b5t/OzKVqvLJHOVTE7aZARsqrdQdiAmd9bbArFIoaLnvHYvArPprLU9clfhQJrAXi484t4eK2h2lMtiitmHph9WpcTxYiDdEg4fxlA2yYDkQZGcjoNPFTBEP5RpmXMcNV7/Zb2U0rJIOPrFqIwZ5yG79Kic8ajPdtmGdVpYec/OlR1Oer0kfakU2RImTzJPZeGnXy/hvcQErAqgpzTLn7dz2YaWMYdcLm6HOmZl1nj3LauWmRcCB8cxM34GjFXMpNm+6Ey8pmgoCMGLdBjL5/xqT6PbQxbYgUJNXMc51qMX8C3lkX/muQD8pDaj5kLXJhYkCgVGwcoC+hbJAOYSry4NQ8aTsFmjyTkaTE= cte@monstera" > /home/vscode/.ssh/authorized_keys
46+
RUN chown -R vscode:vscode /home/vscode/.ssh
47+
RUN chmod 700 /home/vscode/.ssh
48+
RUN chmod 600 /home/vscode/.ssh/authorized_keys
49+
50+
COPY benchmark/entrypoint.sh /usr/local/bin/
51+
COPY --chown=vscode:vscode . /home/vscode/repo
52+
53+
WORKDIR /home/vscode/repo
54+
RUN npm run install:all
55+
56+
WORKDIR /home/vscode/repo/benchmark
57+
RUN git clone https://github.com/cte/Roo-Code-Benchmark.git exercises
58+
RUN npm run build
59+
60+
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
61+
CMD ["/bin/bash"]

‎benchmark/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Benchmark Harness
2+
3+
Configure ENV vars (OpenRouter, PostHog, etc):
4+
5+
```sh
6+
cp .env.local.sample .env.local
7+
# Update ENV vars as needed.
8+
```
9+
10+
Build and run a Docker image with the development environment needed to run the
11+
benchmarks (C++, Go, Java, Node.js, Python & Rust):
12+
13+
```sh
14+
npm run docker:start
15+
```
16+
17+
Run an exercise:
18+
19+
```sh
20+
npm run docker:benchmark -- -e exercises/javascript/binary
21+
```

‎benchmark/entrypoint.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
sudo /usr/sbin/sshd
4+
exec "$@"

0 commit comments

Comments
 (0)