Skip to content

Commit 97f4303

Browse files
everpcpceverpcpc
authored
Merge pull request #10377 from everpcpc/feat-bench
ci(benchmark): run s3 clickbench with cloud
2 parents c2b403d + f8abd29 commit 97f4303

File tree

8 files changed

+429
-139
lines changed

8 files changed

+429
-139
lines changed

‎.github/actions/benchmark_cloud/action.yml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
name: "Benchmark Cloud"
2+
description: "Run benchmark for S3 with cloud storage"
3+
inputs:
4+
sha:
5+
description: "commit sha"
6+
required: true
7+
run_id:
8+
description: "benchmark run id"
9+
required: true
10+
dataset:
11+
description: "hits/tpch"
12+
required: true
13+
source:
14+
description: "pr/release"
15+
required: true
16+
source_id:
17+
description: "pr_id/release_tag"
18+
required: true
19+
size:
20+
description: "Small/Medium/Large"
21+
required: true
22+
image_tag:
23+
description: "Databend image tag"
24+
required: true
25+
cloud_email:
26+
description: "Benchmark cloud email"
27+
required: true
28+
cloud_password:
29+
description: "Benchmark cloud password"
30+
required: true
31+
cloud_org:
32+
description: "Benchmark cloud org"
33+
required: true
34+
cloud_endpoint:
35+
description: "Benchmark cloud endpoint"
36+
required: true
37+
38+
runs:
39+
using: "composite"
40+
steps:
41+
- uses: ./.github/actions/setup_bendsql
42+
43+
- name: Install script dependencies
44+
shell: bash
45+
run: |
46+
sudo apt-get update -y
47+
sudo apt-get install -y jq bc
48+
49+
- name: Run Benchmark
50+
working-directory: benchmark/clickbench
51+
env:
52+
BENCHMARK_ID: ${{ inputs.run_id }}
53+
BENCHMARK_DATASET: ${{ inputs.dataset }}
54+
BENCHMARK_SIZE: ${{ inputs.size }}
55+
BENCHMARK_IMAGE_TAG: ${{ inputs.image_tag }}
56+
CLOUD_EMAIL: ${{ inputs.cloud_email }}
57+
CLOUD_PASSWORD: ${{ inputs.cloud_password }}
58+
CLOUD_ORG: ${{ inputs.cloud_org }}
59+
CLOUD_ENDPOINT: ${{ inputs.cloud_endpoint }}
60+
shell: bash
61+
run: |
62+
./benchmark_cloud.sh
63+
64+
- name: Prepare Metadata
65+
working-directory: benchmark/clickbench
66+
shell: bash
67+
run: |
68+
jq ".date = \"$(date -u +%Y-%m-%d)\"" <result.json >result.json.tmp && mv result.json.tmp result.json
69+
case ${{ inputs.source }} in
70+
pr)
71+
BENCHMARK_SYSTEM="Databend(PR#${{ inputs.source_id }})"
72+
BENCHMARK_TYPE="PR"
73+
;;
74+
release)
75+
BENCHMARK_SYSTEM="Databend(Release@${{ inputs.source_id }})"
76+
BENCHMARK_TYPE="Release"
77+
;;
78+
*)
79+
echo "Unspported benchmark source: ${{ inputs.source }}"
80+
exit 1
81+
esac
82+
jq ".system = \"${BENCHMARK_SYSTEM}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
83+
84+
case ${{ inputs.size }} in
85+
Medium)
86+
BENCHMARK_CLUSTER_SIZE=16
87+
;;
88+
Large)
89+
BENCHMARK_CLUSTER_SIZE=64
90+
;;
91+
*)
92+
echo "Unspported benchmark size: ${{ inputs.size }}"
93+
exit 1
94+
esac
95+
jq ".cluster_size = \"${BENCHMARK_CLUSTER_SIZE}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
96+
jq ".machine = \"${{ inputs.size }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
97+
jq ".tags = [\"${BENCHMARK_TYPE}\", \"serverless\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
98+
jq ".comment = \"commit:${{ inputs.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
99+
mv result.json result-${{ inputs.dataset }}-cloud.json
100+
101+
- name: Upload artifact
102+
uses: actions/upload-artifact@v3
103+
with:
104+
name: benchmark-${{ inputs.dataset }}-cloud-${{ inputs.size }}
105+
path: benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json

.github/actions/benchmark/action.yml renamed to .github/actions/benchmark_local/action.yml

Lines changed: 17 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
name: "Benchmark"
2-
description: "Run benchmark"
1+
name: "Benchmark Local"
2+
description: "Run benchmark for local filesystem"
33
inputs:
4-
storage:
5-
description: "s3/fs"
4+
sha:
5+
description: "commit sha"
6+
required: true
7+
run_id:
8+
description: "benchmark run id"
69
required: true
710
dataset:
811
description: "hits/tpch"
@@ -20,7 +23,7 @@ runs:
2023
uses: ./.github/actions/artifact_download
2124
with:
2225
profile: release
23-
sha: ${{ github.sha }}
26+
sha: ${{ inputs.sha }}
2427
target: x86_64-unknown-linux-gnu
2528

2629
- name: Setup Databend Binary
@@ -31,14 +34,7 @@ runs:
3134
databend-query --version
3235
databend-meta --version
3336
34-
- name: Setup BendSQL
35-
shell: bash
36-
run: |
37-
curl -fsSLo ./bendsql.tar.gz https://github.com/databendcloud/bendsql/releases/download/v0.3.5/bendsql-linux-amd64.tar.gz
38-
tar -xvzf ./bendsql.tar.gz
39-
chmod +x ./linux-amd64/bendsql
40-
sudo cp ./linux-amd64/bendsql /usr/local/bin/bendsql
41-
bendsql version
37+
- uses: ./.github/actions/setup_bendsql
4238

4339
- name: Install script dependencies
4440
shell: bash
@@ -49,29 +45,16 @@ runs:
4945
- name: Run Benchmark
5046
working-directory: benchmark/clickbench
5147
env:
52-
BENCHMARK_ID: ${{ github.run_id }}
53-
BENCHMARK_STORAGE: ${{ inputs.storage }}
48+
BENCHMARK_ID: ${{ inputs.run_id }}
5449
BENCHMARK_DATASET: ${{ inputs.dataset }}
5550
shell: bash
5651
run: |
57-
./benchmark.sh
52+
./benchmark_local.sh
5853
5954
- name: Prepare Metadata
6055
working-directory: benchmark/clickbench
6156
shell: bash
6257
run: |
63-
case ${{ inputs.storage }} in
64-
s3)
65-
BENCHMARK_STORAGE="s3"
66-
;;
67-
fs)
68-
BENCHMARK_STORAGE="gp3"
69-
;;
70-
*)
71-
echo "Unspported benchmark storage: ${{ inputs.storage }}"
72-
exit 1
73-
esac
74-
7558
case ${{ inputs.source }} in
7659
pr)
7760
BENCHMARK_SYSTEM="Databend(PR#${{ inputs.source_id }})"
@@ -87,18 +70,18 @@ runs:
8770
esac
8871
jq ".system = \"${BENCHMARK_SYSTEM}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
8972
jq ".date = \"$(date -u +%Y-%m-%d)\"" <result.json >result.json.tmp && mv result.json.tmp result.json
90-
jq ".machine = \"c5.4xlarge,${BENCHMARK_STORAGE}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
73+
jq ".machine = \"c5.4xlarge,gp3\"" <result.json >result.json.tmp && mv result.json.tmp result.json
9174
jq ".cluster_size = 1" <result.json >result.json.tmp && mv result.json.tmp result.json
92-
jq ".comment = \"commit:${{ github.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
93-
jq ".tags = [\"${BENCHMARK_TYPE}\", \"${BENCHMARK_STORAGE}\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
75+
jq ".comment = \"commit:${{ inputs.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
76+
jq ".tags = [\"${BENCHMARK_TYPE}\", \"gp3\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
9477
jq ".extra.${{ inputs.source }} = \"${{ inputs.source_id }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
95-
mv result.json result-${{ inputs.dataset }}-${{ inputs.storage }}.json
78+
mv result.json result-${{ inputs.dataset }}-local.json
9679
9780
- name: Upload artifact
9881
uses: actions/upload-artifact@v3
9982
with:
100-
name: benchmark-${{ inputs.dataset }}-${{ inputs.storage }}
101-
path: benchmark/clickbench/result-${{ inputs.dataset }}-${{ inputs.storage }}.json
83+
name: benchmark-${{ inputs.dataset }}-local
84+
path: benchmark/clickbench/result-${{ inputs.dataset }}-local.json
10285

10386
- name: Clean test data
10487
working-directory: benchmark/clickbench

‎.github/actions/setup_bendsql/action.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: "Setup BendSQL"
2+
description: "Setup BendSQL for CI"
3+
inputs:
4+
version:
5+
description: "bendsql version"
6+
required: false
7+
default: v0.3.5
8+
runs:
9+
using: "composite"
10+
steps:
11+
- name: Download and Install
12+
shell: bash
13+
run: |
14+
curl -fsSLo ./bendsql.tar.gz https://github.com/databendcloud/bendsql/releases/download/${{ inputs.version }}/bendsql-linux-amd64.tar.gz
15+
tar -xvzf ./bendsql.tar.gz
16+
chmod +x ./linux-amd64/bendsql
17+
sudo cp ./linux-amd64/bendsql /usr/local/bin/bendsql
18+
bendsql version

‎.github/workflows/benchmark.yml‎

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -47,21 +47,6 @@ jobs:
4747
with:
4848
target: x86_64-unknown-linux-gnu
4949
profile: release
50-
51-
clickbench:
52-
needs: build_release
53-
timeout-minutes: 30
54-
runs-on: [self-hosted, X64, Linux, perf]
55-
strategy:
56-
matrix:
57-
storage:
58-
- s3
59-
- fs
60-
dataset:
61-
- hits
62-
- tpch
63-
steps:
64-
- uses: actions/checkout@v3
6550
- name: Get benchmark source
6651
run: |
6752
case ${{ github.event_name }} in
@@ -78,9 +63,15 @@ jobs:
7863
exit 1
7964
;;
8065
esac
81-
- uses: ./.github/actions/benchmark
66+
- name: Generate benchmark metadata
67+
run: |
68+
echo '{}' > metadata.json
69+
jq ".sha = \"${{ github.sha }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
70+
jq ".run_id = \"${{ github.run_id }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
71+
jq ".source = \"${{ env.BENCHMARK_SOURCE }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
72+
jq ".source_id = \"${{ env.BENCHMARK_SOURCE_ID }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
73+
- name: Upload artifact
74+
uses: actions/upload-artifact@v3
8275
with:
83-
storage: ${{ matrix.storage }}
84-
dataset: ${{ matrix.dataset }}
85-
source: ${{ env.BENCHMARK_SOURCE }}
86-
source_id: ${{ env.BENCHMARK_SOURCE_ID }}
76+
name: benchmark-metadata
77+
path: metadata.json

‎.github/workflows/release.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -343,7 +343,6 @@ jobs:
343343
platforms: linux/amd64,linux/arm64
344344
context: .
345345
file: ./docker/Dockerfile
346-
build-args: VERSION=${{ needs.create_release.outputs.version }}
347346
- name: Update repo description
348347
uses: peter-evans/dockerhub-description@v3
349348
with:
@@ -416,7 +415,6 @@ jobs:
416415
platforms: linux/amd64,linux/arm64
417416
context: .
418417
file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile
419-
build-args: VERSION=${{ needs.create_release.outputs.version }}
420418

421419
release_packages:
422420
name: release packages

0 commit comments

Comments
 (0)