Skip to content

Commit d3e2b54

Browse files
TmonsterTmonster
committed
actually run the benchmark now
1 parent d45378a commit d3e2b54

File tree

2 files changed

+87
-15
lines changed

2 files changed

+87
-15
lines changed

‎.github/workflows/RunBenchmark.yml‎

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ env:
1111
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
1212
gh_issue_repo: duckdblabs/db-benchmark
1313

14-
# TODO add AWS ACCESS KEYS and such
1514
# start-aws-machine:
1615
# name: Start aws-small-machine
1716
# runs-on: ubuntu-latest
@@ -40,7 +39,6 @@ jobs:
4039
CC: gcc-10
4140
CXX: g++-10
4241
GEN: ninja
43-
MACHINE_TYPE: c6id.4xlarge
4442
runs-on: self-hosted
4543
steps:
4644
- uses: actions/checkout@v4
@@ -60,13 +58,13 @@ jobs:
6058
run: |
6159
git checkout add_trigger_to_run_the_benchmark
6260
63-
# - name: Install or Upgrade all solutions
64-
# shell: bash
65-
# working-directory: /var/lib/mount/db-benchmark-metal
66-
# run: |
67-
# python3 _setup_utils/install_all_solutions.py all --exclude clickhouse
68-
# # installing/updating clickhouse needs sudo priviledges
69-
# sudo python3 _setup_utils/install_all_solutions.py clickhouse
61+
- name: Install or Upgrade all solutions
62+
shell: bash
63+
working-directory: /var/lib/mount/db-benchmark-metal
64+
run: |
65+
python3 _setup_utils/install_all_solutions.py all --exclude clickhouse
66+
# installing/updating clickhouse needs sudo priviledges
67+
sudo python3 _setup_utils/install_all_solutions.py clickhouse
7068
7169
- name: Modify run.conf to only have new versions
7270
shell: bash
@@ -83,9 +81,14 @@ jobs:
8381
shell: bash
8482
working-directory: /var/lib/mount/db-benchmark-metal
8583
run: |
86-
echo "small edit" >> run.conf
87-
88-
- name: Commit files
84+
ncores=`python3 -c 'import multiprocessing as mp; print(mp.cpu_count())'`
85+
if [ $ncores -eq 16 ]; then export MACHINE_TYPE="c6id.4xlarge"; fi
86+
if [ $ncores -eq 32 ]; then export MACHINE_TYPE="c6id.4xlarge"; fi
87+
if [ $ncores -eq 128 ]; then export MACHINE_TYPE="c6id.metal"; fi
88+
MACHINE_TYPE=$MACHINE_TYPE ./_run/run_small_medium.sh
89+
90+
91+
- name: Commit updates
8992
shell: bash
9093
working-directory: /var/lib/mount/db-benchmark-metal
9194
run: |
@@ -101,10 +104,10 @@ jobs:
101104
working-directory: /var/lib/mount/db-benchmark-metal
102105
run: |
103106
mkdir -p out
104-
echo "neato" > out/whatever.txt
107+
echo "guarantee not empty dir" > out/guarantee.txt
105108
zip -r out-dir.zip out/
106109
107-
- name: new branch name check
110+
- name: name new branch
108111
shell: bash
109112
run: |
110113
echo "new_branch_name=results-`date +%Y-%m-%d-%Hh%Mm`" >> $GITHUB_ENV
@@ -124,3 +127,20 @@ jobs:
124127
github_token: ${{ secrets.GITHUB_TOKEN }}
125128
branch: ${{ env.branch_name }}
126129

130+
# shutdown:
131+
# name: shut down
132+
# if: always()
133+
# runs-on: ubuntu-latest
134+
# needs:
135+
# - start-runner
136+
# - run-benchmark
137+
138+
# steps:
139+
# - name: shutdown
140+
# shell: bash
141+
# env:
142+
# AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
143+
# AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
144+
# AWS_DEFAULT_REGION: us-east-1
145+
# run: aws ec2 stop-instances --instance-id ${{ env.instance_id }}
146+

‎_control/skipped_benchmarks.csv‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,57 @@
11
solution,task,data,machine_type
2+
juliads,groupby,G1_1e9_1e2_0_0,c6id.4xlarge
3+
juliads,groupby,G1_1e9_1e1_0_0,c6id.4xlarge
4+
juliads,groupby,G1_1e9_2e0_0_0,c6id.4xlarge
5+
juliads,groupby,G1_1e9_1e2_0_1,c6id.4xlarge
6+
juliads,groupby,G1_1e9_1e2_5_0,c6id.4xlarge
7+
juliadf,groupby,G1_1e9_1e2_0_0,c6id.4xlarge
8+
juliadf,groupby,G1_1e9_1e1_0_0,c6id.4xlarge
9+
juliadf,groupby,G1_1e9_2e0_0_0,c6id.4xlarge
10+
juliadf,groupby,G1_1e9_1e2_0_1,c6id.4xlarge
11+
juliadf,groupby,G1_1e9_1e2_5_0,c6id.4xlarge
12+
R-arrow,groupby,G1_1e9_1e2_0_0,c6id.4xlarge
13+
R-arrow,groupby,G1_1e9_1e1_0_0,c6id.4xlarge
14+
R-arrow,groupby,G1_1e9_2e0_0_0,c6id.4xlarge
15+
R-arrow,groupby,G1_1e9_1e2_0_1,c6id.4xlarge
16+
R-arrow,groupby,G1_1e9_1e2_5_0,c6id.4xlarge
17+
dplyr,groupby,G1_1e9_1e2_0_0,c6id.4xlarge
18+
dplyr,groupby,G1_1e9_1e1_0_0,c6id.4xlarge
19+
dplyr,groupby,G1_1e9_2e0_0_0,c6id.4xlarge
20+
dplyr,groupby,G1_1e9_1e2_0_1,c6id.4xlarge
21+
dplyr,groupby,G1_1e9_1e2_5_0,c6id.4xlarge
22+
pandas,groupby,G1_1e9_1e2_0_0,c6id.4xlarge
23+
pandas,groupby,G1_1e9_1e1_0_0,c6id.4xlarge
24+
pandas,groupby,G1_1e9_2e0_0_0,c6id.4xlarge
25+
pandas,groupby,G1_1e9_1e2_0_1,c6id.4xlarge
26+
pandas,groupby,G1_1e9_1e2_5_0,c6id.4xlarge
27+
pydatatable,groupby,G1_1e9_1e2_0_0,c6id.4xlarge
28+
pydatatable,groupby,G1_1e9_1e1_0_0,c6id.4xlarge
29+
pydatatable,groupby,G1_1e9_2e0_0_0,c6id.4xlarge
30+
pydatatable,groupby,G1_1e9_1e2_0_1,c6id.4xlarge
31+
pydatatable,groupby,G1_1e9_1e2_5_0,c6id.4xlarge
32+
spark,groupby,G1_1e9_1e2_0_0,c6id.4xlarge
33+
spark,groupby,G1_1e9_1e1_0_0,c6id.4xlarge
34+
spark,groupby,G1_1e9_2e0_0_0,c6id.4xlarge
35+
spark,groupby,G1_1e9_1e2_0_1,c6id.4xlarge
36+
spark,groupby,G1_1e9_1e2_5_0,c6id.4xlarge
37+
datafusion,groupby,G1_1e9_1e2_0_0,c6id.4xlarge
38+
datafusion,groupby,G1_1e9_1e1_0_0,c6id.4xlarge
39+
datafusion,groupby,G1_1e9_2e0_0_0,c6id.4xlarge
40+
datafusion,groupby,G1_1e9_1e2_0_1,c6id.4xlarge
41+
datafusion,groupby,G1_1e9_1e2_5_0,c6id.4xlarge
242
datafusion,join,J1_1e8_NA_0_0,c6id.4xlarge
343
datafusion,join,J1_1e8_NA_5_0,c6id.4xlarge
444
datafusion,join,J1_1e8_NA_0_1,c6id.4xlarge
5-
datafusion,join,J1_1e9_NA_0_0,c6id.4xlarge
45+
datafusion,join,J1_1e9_NA_0_0,c6id.4xlarge
46+
R-arrow,join,J1_1e9_NA_0_0,c6id.4xlarge
47+
dask,join,J1_1e9_NA_0_0,c6id.4xlarge
48+
datatable,join,J1_1e9_NA_0_0,c6id.4xlarge
49+
juliadf,join,J1_1e9_NA_0_0,c6id.4xlarge
50+
juliads,join,J1_1e9_NA_0_0,c6id.4xlarge
51+
pandas,join,J1_1e9_NA_0_0,c6id.4xlarge
52+
collapse,join,J1_1e9_NA_0_0,c6id.4xlarge
53+
polars,join,J1_1e9_NA_0_0,c6id.4xlarge
54+
pydatatable,join,J1_1e9_NA_0_0,c6id.4xlarge
55+
spark,join,J1_1e9_NA_0_0,c6id.4xlarge
56+
clickhouse,join,J1_1e9_NA_0_0,c6id.4xlarge
57+

0 commit comments

Comments
 (0)