Skip to content

Commit cd60b34

Browse files
ctecte
committed
More progress
1 parent a770ff5 commit cd60b34

File tree

5 files changed

+147
-56
lines changed

5 files changed

+147
-56
lines changed

‎.tool-versions‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
nodejs v20.18.1

‎benchmark/.env.sample‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
11
BENCHMARKS_DB_PATH=file:/tmp/benchmarks.db
2-
OPENROUTER_API_KEY=sk-or-v1-...

‎benchmark/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,6 @@ yarn-error.log*
3737
# Misc
3838
.DS_Store
3939
*.pem
40+
41+
# Evals
42+
evals

‎benchmark/.tool-versions‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
nodejs v20.18.1
2+
python 3.13.2
3+
golang 1.24.2
4+
rust 1.85.1

‎benchmark/scripts/setup.sh‎

Lines changed: 139 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
1-
#!/bin/bash
1+
#!/usr/bin/env bash
22

33
if [[ "$(uname -s)" != "Darwin" ]]; then
44
echo "Only macOS is currently supported."
55
exit 1
66
fi
77

8-
options=("nodejs" "python" "golang" "rust")
9-
binaries=("node" "python" "go" "rustc")
10-
choices[0]="*"
8+
options=("nodejs" "python" "golang" "rust" "java")
9+
binaries=("node" "python" "go" "rustc" "javac")
10+
11+
declare -A has_asdf_plugin
12+
has_asdf_plugin=([nodejs]=true [python]=true [golang]=true [rust]=true [java]=false)
13+
14+
for i in "${!options[@]}"; do
15+
choices[i]="*"
16+
done
1117

1218
menu() {
1319
echo -e "\nWhich eval types would you like to support?\n"
@@ -70,8 +76,8 @@ if ! command -v brew &>/dev/null; then
7076
read -p "Homebrew (https://brew.sh) is required. Install it? (Y/n): " install_brew
7177

7278
if [[ "$install_brew" =~ ^[Yy]|^$ ]]; then
73-
echo "Installing Homebrew..."
74-
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
79+
echo "☕ Installing Homebrew..."
80+
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" || exit 1
7581
# Can be undone with:
7682
# /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/uninstall.sh)" && sudo rm -rvf /opt/homebrew
7783

@@ -87,7 +93,8 @@ if ! command -v brew &>/dev/null; then
8793
eval "$(/opt/homebrew/bin/brew shellenv)"
8894
fi
8995

90-
echo "✅ Homebrew is installed"
96+
BREW_VERSION=$(brew --version)
97+
echo "✅ Homebrew is installed ($BREW_VERSION)"
9198
else
9299
exit 1
93100
fi
@@ -105,12 +112,13 @@ if ! command -v asdf &>/dev/null; then
105112
fi
106113

107114
read -p "asdf (https://asdf-vm.com) is required. Install it? (Y/n): " install_asdf
108-
# Can be undone with:
109-
# rm -rvf ~/.asdf
110115

111116
if [[ "$install_asdf" =~ ^[Yy]|^$ ]]; then
112117
echo "Installing asdf..."
113-
brew install asdf
118+
brew install asdf || exit 1
119+
# Can be undone with:
120+
# brew uninstall asdf
121+
# rm -rvf ~/.asdf
114122

115123
. "$ASDF_PATH"
116124

@@ -120,7 +128,8 @@ if ! command -v asdf &>/dev/null; then
120128
echo '[[ -s "/opt/homebrew/bin/brew" ]] && [[ -s "$(brew --prefix asdf)/libexec/asdf.sh" ]] && source "$(brew --prefix asdf)/libexec/asdf.sh"' >>~/.bash_profile
121129
fi
122130

123-
echo "✅ asdf is installed"
131+
ASDF_VERSION=$(asdf --version)
132+
echo "✅ asdf is installed ($ASDF_VERSION)"
124133
else
125134
exit 1
126135
fi
@@ -129,71 +138,146 @@ else
129138
echo "✅ asdf is installed ($ASDF_VERSION)"
130139
fi
131140

141+
if ! command -v gh &>/dev/null; then
142+
read -p "GitHub cli is needed to submit evals results. Install it? (Y/n): " install_gh
143+
144+
if [[ "$install_gh" =~ ^[Yy]|^$ ]]; then
145+
brew install gh || exit 1
146+
GH_VERSION=$(gh --version | head -n 1)
147+
echo "✅ gh is installed ($GH_VERSION)"
148+
gh auth status || gh auth login -w -p https
149+
fi
150+
else
151+
GH_VERSION=$(gh --version | head -n 1)
152+
echo "✅ gh is installed ($GH_VERSION)"
153+
fi
154+
132155
for i in "${!options[@]}"; do
133156
[[ "${choices[i]}" ]] || continue
134157

135158
plugin="${options[$i]}"
136159
binary="${binaries[$i]}"
137160

138-
if ! asdf plugin list | grep -q "^${plugin}$" && ! command -v "${binary[$i]}" &>/dev/null; then
139-
asdf plugin add "${plugin}"
161+
if [[ "${has_asdf_plugin[$plugin]}" == "true" ]]; then
162+
missing_plugin=$(! asdf plugin list | grep -q "^${plugin}$")
140163

141-
if ! asdf plugin list | grep -q "^${plugin}$"; then
142-
echo "Failed to install ${plugin} asdf plugin. Please install it manually."
143-
exit 1
144-
else
164+
if [[ "$missing_plugin" == true ]] && ! command -v "${binary}" &>/dev/null; then
165+
echo "Installing ${plugin} asdf plugin..."
166+
asdf plugin add "${plugin}" || exit 1
145167
echo "✅ asdf ${plugin} plugin installed"
146168
fi
147169
fi
148170

149-
if [[ "${plugin}" == "nodejs" ]] && ! command -v node &>/dev/null; then
150-
asdf install nodejs v20.18.1
151-
asdf set nodejs v20.18.1
152-
NODE_VERSION=$(node --version)
153-
echo "✅ Node.js is installed ($NODE_VERSION)"
154-
elif [[ "${plugin}" == "nodejs" ]]; then
155-
NODE_VERSION=$(node --version)
156-
echo "✅ Node.js is installed ($NODE_VERSION)"
157-
fi
171+
case "${plugin}" in
172+
"nodejs")
173+
if ! command -v node &>/dev/null; then
174+
asdf install nodejs v20.18.1 || exit 1
175+
asdf set nodejs v20.18.1 || exit 1
176+
NODE_VERSION=$(node --version)
177+
echo "✅ Node.js is installed ($NODE_VERSION)"
178+
else
179+
NODE_VERSION=$(node --version)
180+
echo "✅ Node.js is installed ($NODE_VERSION)"
181+
fi
158182

159-
if [[ "${plugin}" == "python" ]] && ! command -v python &>/dev/null; then
160-
asdf install python 3.13.2
161-
asdf set python 3.13.2
162-
PYTHON_VERSION=$(python --version)
163-
echo "✅ Python is installed ($PYTHON_VERSION)"
164-
elif [[ "${plugin}" == "python" ]]; then
165-
PYTHON_VERSION=$(python --version)
166-
echo "✅ Python is installed ($PYTHON_VERSION)"
167-
fi
183+
if [[ $(node --version) != "v20.18.1" ]]; then
184+
NODE_VERSION=$(node --version)
185+
echo "🚨 You have the wrong version of node installed ($NODE_VERSION)."
186+
echo "If you are using nvm then run 'nvm install' to install the version specified by the repo's .nvmrc."
187+
exit 1
188+
fi
189+
;;
190+
191+
"python")
192+
if ! command -v python &>/dev/null; then
193+
asdf install python 3.13.2 || exit 1
194+
asdf set python 3.13.2 || exit 1
195+
PYTHON_VERSION=$(python --version)
196+
echo "✅ Python is installed ($PYTHON_VERSION)"
197+
else
198+
PYTHON_VERSION=$(python --version)
199+
echo "✅ Python is installed ($PYTHON_VERSION)"
200+
fi
168201

169-
if [[ "${plugin}" == "golang" ]] && ! command -v go &>/dev/null; then
170-
asdf install golang 1.24.2
171-
asdf set golang 1.24.2
172-
GO_VERSION=$(go version)
173-
echo "✅ Go is installed ($GO_VERSION)"
174-
elif [[ "${plugin}" == "golang" ]]; then
175-
GO_VERSION=$(go version)
176-
echo "✅ Go is installed ($GO_VERSION)"
177-
fi
202+
if ! command -v uv &>/dev/null; then
203+
brew install uv || exit 1
204+
UV_VERSION=$(uv --version)
205+
echo "✅ uv is installed ($UV_VERSION)"
206+
else
207+
UV_VERSION=$(uv --version)
208+
echo "✅ uv is installed ($UV_VERSION)"
209+
fi
210+
;;
211+
212+
"golang")
213+
if ! command -v go &>/dev/null; then
214+
asdf install golang 1.24.2 || exit 1
215+
asdf set golang 1.24.2 || exit 1
216+
GO_VERSION=$(go version)
217+
echo "✅ Go is installed ($GO_VERSION)"
218+
else
219+
GO_VERSION=$(go version)
220+
echo "✅ Go is installed ($GO_VERSION)"
221+
fi
222+
;;
223+
224+
"rust")
225+
if ! command -v rustc &>/dev/null; then
226+
asdf install rust 1.85.1 || exit 1
227+
asdf set rust 1.85.1 || exit 1
228+
RUST_VERSION=$(rustc --version)
229+
echo "✅ Rust is installed ($RUST_VERSION)"
230+
else
231+
RUST_VERSION=$(rustc --version)
232+
echo "✅ Rust is installed ($RUST_VERSION)"
233+
fi
234+
;;
178235

179-
if [[ "${plugin}" == "rust" ]] && ! command -v rustc &>/dev/null; then
180-
asdf install rust 1.85.1
181-
asdf set rust 1.85.1
182-
RUST_VERSION=$(rustc --version)
183-
echo "✅ Rust is installed ($RUST_VERSION)"
184-
elif [[ "${plugin}" == "rust" ]]; then
185-
RUST_VERSION=$(rustc --version)
186-
echo "✅ Rust is installed ($RUST_VERSION)"
187-
fi
236+
"java")
237+
if ! command -v javac &>/dev/null; then
238+
brew install openjdk@17 || exit 1
239+
JAVA_VERSION=$(java --version | head -n 1)
240+
echo "✅ Java is installed ($JAVA_VERSION)"
241+
else
242+
JAVA_VERSION=$(java --version | head -n 1)
243+
echo "✅ Java is installed ($JAVA_VERSION)"
244+
fi
245+
;;
246+
esac
188247
done
189248

190249
if ! command -v pnpm &>/dev/null; then
191-
brew install pnpm
250+
brew install pnpm || exit 1
192251
PNPM_VERSION=$(pnpm --version)
193252
echo "✅ pnpm is installed ($PNPM_VERSION)"
194253
else
195254
PNPM_VERSION=$(pnpm --version)
196255
echo "✅ pnpm is installed ($PNPM_VERSION)"
197256
fi
198257

199-
pnpm install
258+
pnpm install || exit 1
259+
260+
if [[ ! -d "evals" ]]; then
261+
if gh auth status &>/dev/null; then
262+
read -p "Would you like to be able to share eval results? (Y/n): " fork_evals
263+
264+
if [[ "$fork_evals" =~ ^[Yy]|^$ ]]; then
265+
gh repo fork cte/evals || exit 1
266+
else
267+
gh repo clone cte/evals || exit 1
268+
fi
269+
else
270+
git clone https://github.com/cte/evals.git || exit 1
271+
fi
272+
fi
273+
274+
if [[ ! -s .env ]]; then
275+
cp .env.sample .env || exit 1
276+
fi
277+
278+
if ! grep -q "OPENROUTER_API_KEY" .env; then
279+
read -p "Enter your OpenRouter API Key (sk-or-v1-...): " openrouter_api_key
280+
echo "Validating OpenRouter API Key..."
281+
curl --silent --fail https://openrouter.ai/api/v1/key -H "Authorization: Bearer $openrouter_api_key" | jq || exit 1
282+
echo "OPENROUTER_API_KEY=$openrouter_api_key" >> .env
283+
fi

0 commit comments

Comments
 (0)