Skip to content

Commit 6c63cf9

Browse files
dankingdanking
authored
feat: store benchmarks in s3 (#1650)
Preview the new page here: https://spiraldb.github.io/vortex/dev/bench2/
1 parent dc56eac commit 6c63cf9

File tree

16 files changed

+872
-180
lines changed

16 files changed

+872
-180
lines changed

‎.github/workflows/bench-pr.yml‎

Lines changed: 90 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ jobs:
3232
name: Random Access
3333
- id: compress
3434
name: Vortex Compression
35-
3635
runs-on: self-hosted
3736
if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
3837
steps:
@@ -60,45 +59,44 @@ jobs:
6059
RUSTFLAGS: '-C target-cpu=native'
6160
run: |
6261
cargo install cargo-criterion
63-
64-
cargo criterion --bench ${{ matrix.benchmark.id }} --message-format=json 2>&1 | tee out.json
65-
66-
cat out.json
67-
6862
sudo apt-get update && sudo apt-get install -y jq
6963
70-
jq --raw-input --compact-output '
71-
fromjson?
72-
| [ (if .mean != null then {name: .id, value: .mean.estimate, unit: .unit, range: ((.mean.upper_bound - .mean.lower_bound) / 2) } else {} end),
73-
(if .throughput != null then {name: (.id + " throughput"), value: .throughput[].per_iteration, unit: .throughput[].unit, range: 0} else {} end),
74-
{name, value, unit, range} ]
75-
| .[]
76-
| select(.value != null)
77-
' \
78-
out.json \
79-
| jq --slurp --compact-output '.' >${{ matrix.benchmark.id }}.json
80-
81-
cat ${{ matrix.benchmark.id }}.json
82-
- name: Store benchmark result
83-
if: '!cancelled()'
84-
uses: benchmark-action/github-action-benchmark@v1
64+
cargo criterion \
65+
--bench ${{ matrix.benchmark.id }} \
66+
--message-format=json \
67+
> ${{ matrix.benchmark.id }}-raw.json
68+
69+
cat ${{ matrix.benchmark.id }}-raw.json \
70+
| bash scripts/coerce-criterion-json.sh \
71+
> ${{ matrix.benchmark.id }}.json
72+
73+
- name: Setup AWS CLI
74+
uses: aws-actions/configure-aws-credentials@v1
8575
with:
86-
name: ${{ matrix.benchmark.name }}
87-
tool: 'customSmallerIsBetter'
88-
gh-pages-branch: gh-pages-bench
89-
github-token: ${{ secrets.GITHUB_TOKEN }}
90-
output-file-path: ${{ matrix.benchmark.id }}.json
91-
summary-always: true
92-
comment-always: true
93-
auto-push: false
94-
save-data-file: false
95-
fail-on-alert: false
96-
env:
97-
# AWS Credentials for R2 storage tests
98-
AWS_BUCKET: vortex-test
99-
AWS_ENDPOINT: ${{ secrets.AWS_ENDPOINT }}
100-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
101-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
76+
role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
77+
aws-region: us-east-1
78+
- name: Compare results
79+
shell: bash
80+
run: |
81+
set -Eeu -o pipefail -x
82+
83+
base_commit_sha=${{ github.event.pull_request.base.sha }}
84+
85+
aws s3 cp s3://vortex-benchmark-results-database/data.json - \
86+
| grep $base_commit_sha \
87+
> base.json
88+
89+
echo '# Benchmarks: ${{ matrix.benchmark.id }}' > comment.md
90+
echo '<details>' >> comment.md
91+
echo '<summary>Table of Results</summary>' >> comment.md
92+
uv run python3 scripts/compare-benchmark-jsons.py base.json ${{ matrix.benchmark.id }}.json \
93+
>> comment.md
94+
echo '</details>' >> comment.md
95+
- name: Comment PR
96+
uses: thollander/actions-comment-pull-request@v3
97+
with:
98+
file-path: comment.md
99+
comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
102100
tpch:
103101
needs: label_trigger
104102
runs-on: self-hosted
@@ -126,27 +124,34 @@ jobs:
126124
BENCH_VORTEX_RATIOS: '.*'
127125
RUSTFLAGS: '-C target-cpu=native'
128126
run: |
129-
cargo run --bin tpch_benchmark --release -- --only-vortex -d gh-json -t 1 | tee tpch.json
130-
- name: Store benchmark result
131-
if: '!cancelled()'
132-
uses: benchmark-action/github-action-benchmark@v1
127+
cargo run --bin tpch_benchmark --release -- -d gh-json -t 1 | tee tpch.json
128+
- name: Setup AWS CLI
129+
uses: aws-actions/configure-aws-credentials@v1
133130
with:
134-
name: 'TPC-H'
135-
tool: 'customSmallerIsBetter'
136-
gh-pages-branch: gh-pages-bench
137-
github-token: ${{ secrets.GITHUB_TOKEN }}
138-
output-file-path: tpch.json
139-
summary-always: true
140-
comment-always: true
141-
auto-push: false
142-
save-data-file: false
143-
fail-on-alert: false
144-
env:
145-
# AWS Credentials for R2 storage tests
146-
AWS_BUCKET: vortex-test
147-
AWS_ENDPOINT: ${{ secrets.AWS_ENDPOINT }}
148-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
149-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
131+
role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
132+
aws-region: us-east-1
133+
- name: Compare results
134+
shell: bash
135+
run: |
136+
set -Eeu -o pipefail -x
137+
138+
base_commit_sha=${{ github.event.pull_request.base.sha }}
139+
140+
aws s3 cp s3://vortex-benchmark-results-database/data.json - \
141+
| grep $base_commit_sha \
142+
> base.json
143+
144+
echo '# Benchmarks: TPC-H' > comment.md
145+
echo '<details>' >> comment.md
146+
echo '<summary>Table of Results</summary>' >> comment.md
147+
uv run python3 scripts/compare-benchmark-jsons.py base.json tpch.json \
148+
>> comment.md
149+
echo '</details>' >> comment.md
150+
- name: Comment PR
151+
uses: thollander/actions-comment-pull-request@v3
152+
with:
153+
file-path: comment.md
154+
comment-tag: bench-pr-comment-tpch
150155
clickbench:
151156
needs: label_trigger
152157
runs-on: self-hosted
@@ -168,31 +173,38 @@ jobs:
168173
run: |
169174
echo "TMPDIR=/work" >> $GITHUB_ENV
170175
171-
- name: Run ClickBench benchmark
176+
- name: Run Clickbench benchmark
172177
shell: bash
173178
env:
174179
BENCH_VORTEX_RATIOS: '.*'
175180
RUSTFLAGS: '-C target-cpu=native'
176181
HOME: /home/ci-runner
177182
run: |
178-
cargo run --bin clickbench --release -- --only-vortex -d gh-json | tee clickbench.json
179-
- name: Store benchmark result
180-
if: '!cancelled()'
181-
uses: benchmark-action/github-action-benchmark@v1
183+
cargo run --bin clickbench --release -- -d gh-json | tee clickbench.json
184+
- name: Setup AWS CLI
185+
uses: aws-actions/configure-aws-credentials@v1
182186
with:
183-
name: 'Clickbench'
184-
tool: 'customSmallerIsBetter'
185-
gh-pages-branch: gh-pages-bench
186-
github-token: ${{ secrets.GITHUB_TOKEN }}
187-
output-file-path: clickbench.json
188-
summary-always: true
189-
comment-always: true
190-
auto-push: false
191-
save-data-file: false
192-
fail-on-alert: false
193-
env:
194-
# AWS Credentials for R2 storage tests
195-
AWS_BUCKET: vortex-test
196-
AWS_ENDPOINT: ${{ secrets.AWS_ENDPOINT }}
197-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
198-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
187+
role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
188+
aws-region: us-east-1
189+
- name: Compare results
190+
shell: bash
191+
run: |
192+
set -Eeu -o pipefail -x
193+
194+
base_commit_sha=${{ github.event.pull_request.base.sha }}
195+
196+
aws s3 cp s3://vortex-benchmark-results-database/data.json - \
197+
| grep $base_commit_sha \
198+
> base.json
199+
200+
echo '# Benchmarks: Clickbench' > comment.md
201+
echo '<details>' >> comment.md
202+
echo '<summary>Table of Results</summary>' >> comment.md
203+
uv run python3 scripts/compare-benchmark-jsons.py base.json clickbench.json \
204+
>> comment.md
205+
echo '</details>' >> comment.md
206+
- name: Comment PR
207+
uses: thollander/actions-comment-pull-request@v3
208+
with:
209+
file-path: comment.md
210+
comment-tag: bench-pr-comment-clickbench

‎.github/workflows/bench.yml‎

Lines changed: 47 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,26 @@ on:
55
branches: [ develop ]
66

77
permissions:
8+
id-token: write # enables AWS-GitHub OIDC
89
actions: read
910
contents: write
1011
deployments: write
1112

1213
jobs:
14+
commit-metadata:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- uses: actions/checkout@v4
18+
- name: Setup AWS CLI
19+
uses: aws-actions/configure-aws-credentials@v1
20+
with:
21+
role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
22+
aws-region: us-east-1
23+
- name: Upload Commit Metadata
24+
shell: bash
25+
run: |
26+
sudo apt-get update && sudo apt-get install -y jq
27+
bash scripts/cat-s3.sh vortex-benchmark-results-database commits.json <(bash scripts/commit-json.sh)
1328
bench:
1429
strategy:
1530
matrix:
@@ -46,43 +61,26 @@ jobs:
4661
RUSTFLAGS: '-C target-cpu=native'
4762
run: |
4863
cargo install cargo-criterion
49-
50-
cargo criterion --bench ${{ matrix.benchmark.id }} --message-format=json 2>&1 | tee out.json
51-
52-
cat out.json
53-
5464
sudo apt-get update && sudo apt-get install -y jq
5565
56-
jq --raw-input --compact-output '
57-
fromjson?
58-
| [ (if .mean != null then {name: .id, value: .mean.estimate, unit: .unit, range: ((.mean.upper_bound - .mean.lower_bound) / 2) } else {} end),
59-
(if .throughput != null then {name: (.id + " throughput"), value: .throughput[].per_iteration, unit: .throughput[].unit, range: 0} else {} end),
60-
{name, value, unit, range} ]
61-
| .[]
62-
| select(.value != null)
63-
' \
64-
out.json \
65-
| jq --slurp --compact-output '.' >${{ matrix.benchmark.id }}.json
66+
cargo criterion \
67+
--bench ${{ matrix.benchmark.id }} \
68+
--message-format=json \
69+
> ${{ matrix.benchmark.id }}-raw.json
6670
67-
cat ${{ matrix.benchmark.id }}.json
68-
- name: Store benchmark result
69-
if: '!cancelled()'
70-
uses: benchmark-action/github-action-benchmark@v1
71+
cat ${{ matrix.benchmark.id }}-raw.json \
72+
| bash scripts/coerce-criterion-json.sh \
73+
> ${{ matrix.benchmark.id }}.json
74+
75+
- name: Setup AWS CLI
76+
uses: aws-actions/configure-aws-credentials@v1
7177
with:
72-
name: ${{ matrix.benchmark.name }}
73-
tool: 'customSmallerIsBetter'
74-
gh-pages-branch: gh-pages-bench
75-
github-token: ${{ secrets.GITHUB_TOKEN }}
76-
output-file-path: ${{ matrix.benchmark.id }}.json
77-
summary-always: true
78-
auto-push: true
79-
fail-on-alert: false
80-
env:
81-
# AWS Credentials for R2 storage tests
82-
AWS_BUCKET: vortex-test
83-
AWS_ENDPOINT: ${{ secrets.AWS_ENDPOINT }}
84-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
85-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
78+
role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
79+
aws-region: us-east-1
80+
- name: Upload Benchmark Results
81+
shell: bash
82+
run: |
83+
bash scripts/cat-s3.sh vortex-benchmark-results-database data.json ${{ matrix.benchmark.id }}.json
8684
tpch:
8785
runs-on: self-hosted
8886
steps:
@@ -110,24 +108,15 @@ jobs:
110108
RUSTFLAGS: '-C target-cpu=native'
111109
run: |
112110
cargo run --bin tpch_benchmark --release -- -d gh-json -t 1 | tee tpch.json
113-
- name: Store benchmark result
114-
if: '!cancelled()'
115-
uses: benchmark-action/github-action-benchmark@v1
111+
- name: Setup AWS CLI
112+
uses: aws-actions/configure-aws-credentials@v1
116113
with:
117-
name: 'TPC-H'
118-
tool: 'customSmallerIsBetter'
119-
gh-pages-branch: gh-pages-bench
120-
github-token: ${{ secrets.GITHUB_TOKEN }}
121-
output-file-path: tpch.json
122-
summary-always: true
123-
auto-push: true
124-
fail-on-alert: false
125-
env:
126-
# AWS Credentials for R2 storage tests
127-
AWS_BUCKET: vortex-test
128-
AWS_ENDPOINT: ${{ secrets.AWS_ENDPOINT }}
129-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
130-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
114+
role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
115+
aws-region: us-east-1
116+
- name: Upload Benchmark Results
117+
shell: bash
118+
run: |
119+
bash scripts/cat-s3.sh vortex-benchmark-results-database data.json tpch.json
131120
clickbench:
132121
runs-on: self-hosted
133122
steps:
@@ -156,23 +145,12 @@ jobs:
156145
HOME: /home/ci-runner
157146
run: |
158147
cargo run --bin clickbench --release -- -d gh-json | tee clickbench.json
159-
- name: Store benchmark result
160-
if: '!cancelled()'
161-
uses: benchmark-action/github-action-benchmark@v1
148+
- name: Setup AWS CLI
149+
uses: aws-actions/configure-aws-credentials@v1
162150
with:
163-
name: 'Clickbench'
164-
tool: 'customSmallerIsBetter'
165-
gh-pages-branch: gh-pages-bench
166-
github-token: ${{ secrets.GITHUB_TOKEN }}
167-
output-file-path: clickbench.json
168-
summary-always: true
169-
auto-push: true
170-
fail-on-alert: false
171-
env:
172-
# AWS Credentials for R2 storage tests
173-
AWS_BUCKET: vortex-test
174-
AWS_ENDPOINT: ${{ secrets.AWS_ENDPOINT }}
175-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
176-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
177-
178-
151+
role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
152+
aws-region: us-east-1
153+
- name: Upload Benchmark Results
154+
shell: bash
155+
run: |
156+
bash scripts/cat-s3.sh vortex-benchmark-results-database data.json clickbench.json

0 commit comments

Comments
 (0)