Skip to content

Commit 83c61a7

Browse files
andylokandyandylokandy
committed
Merge branch 'main' of https://github.com/datafuselabs/databend into fixx3
2 parents b7ca3e6 + f7fcd3e commit 83c61a7

File tree

261 files changed

+4597
-4471
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

261 files changed

+4597
-4471
lines changed

‎.github/actions/benchmark_cloud/action.yml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: "Benchmark Cloud"
2+
description: "Run benchmark for S3 with cloud storage"
3+
inputs:
4+
sha:
5+
description: "commit sha"
6+
required: true
7+
run_id:
8+
description: "benchmark run id"
9+
required: true
10+
dataset:
11+
description: "hits/tpch"
12+
required: true
13+
source:
14+
description: "pr/release"
15+
required: true
16+
source_id:
17+
description: "pr_id/release_tag"
18+
required: true
19+
size:
20+
description: "Small/Medium/Large"
21+
required: true
22+
image_tag:
23+
description: "Databend image tag"
24+
required: true
25+
cloud_email:
26+
description: "Benchmark cloud email"
27+
required: true
28+
cloud_password:
29+
description: "Benchmark cloud password"
30+
required: true
31+
cloud_org:
32+
description: "Benchmark cloud org"
33+
required: true
34+
cloud_endpoint:
35+
description: "Benchmark cloud endpoint"
36+
required: true
37+
38+
runs:
39+
using: "composite"
40+
steps:
41+
- name: Install script dependencies
42+
shell: bash
43+
run: |
44+
sudo apt-get update -y
45+
sudo apt-get install -y jq bc
46+
47+
- name: Run Benchmark
48+
working-directory: benchmark/clickbench
49+
env:
50+
BENCHMARK_ID: ${{ inputs.run_id }}
51+
BENCHMARK_DATASET: ${{ inputs.dataset }}
52+
BENCHMARK_SIZE: ${{ inputs.size }}
53+
BENCHMARK_IMAGE_TAG: ${{ inputs.image_tag }}
54+
CLOUD_EMAIL: ${{ inputs.cloud_email }}
55+
CLOUD_PASSWORD: ${{ inputs.cloud_password }}
56+
CLOUD_ORG: ${{ inputs.cloud_org }}
57+
CLOUD_ENDPOINT: ${{ inputs.cloud_endpoint }}
58+
shell: bash
59+
run: |
60+
./benchmark_cloud.sh
61+
62+
- name: Prepare Metadata
63+
working-directory: benchmark/clickbench
64+
shell: bash
65+
run: |
66+
jq ".date = \"$(date -u +%Y-%m-%d)\"" <result.json >result.json.tmp && mv result.json.tmp result.json
67+
case ${{ inputs.source }} in
68+
pr)
69+
BENCHMARK_SYSTEM="Databend(PR#${{ inputs.source_id }})"
70+
BENCHMARK_TYPE="PR"
71+
;;
72+
release)
73+
BENCHMARK_SYSTEM="Databend(Release@${{ inputs.source_id }})"
74+
BENCHMARK_TYPE="Release"
75+
;;
76+
*)
77+
echo "Unspported benchmark source: ${{ inputs.source }}"
78+
exit 1
79+
esac
80+
jq ".system = \"${BENCHMARK_SYSTEM}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
81+
82+
case ${{ inputs.size }} in
83+
Medium)
84+
BENCHMARK_CLUSTER_SIZE=16
85+
;;
86+
Large)
87+
BENCHMARK_CLUSTER_SIZE=64
88+
;;
89+
*)
90+
echo "Unspported benchmark size: ${{ inputs.size }}"
91+
exit 1
92+
esac
93+
jq ".cluster_size = \"${BENCHMARK_CLUSTER_SIZE}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
94+
jq ".machine = \"${{ inputs.size }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
95+
jq ".tags = [\"${BENCHMARK_TYPE}\", \"serverless\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
96+
jq ".comment = \"commit:${{ inputs.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
97+
mv result.json result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json
98+
99+
- name: Upload artifact
100+
uses: actions/upload-artifact@v3
101+
with:
102+
name: benchmark-${{ inputs.dataset }}-cloud-${{ inputs.size }}
103+
path: benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json

.github/actions/benchmark/action.yml renamed to .github/actions/benchmark_local/action.yml

Lines changed: 15 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
name: "Benchmark"
2-
description: "Run benchmark"
1+
name: "Benchmark Local"
2+
description: "Run benchmark for local filesystem"
33
inputs:
4-
storage:
5-
description: "s3/fs"
4+
sha:
5+
description: "commit sha"
6+
required: true
7+
run_id:
8+
description: "benchmark run id"
69
required: true
710
dataset:
811
description: "hits/tpch"
@@ -16,30 +19,6 @@ inputs:
1619
runs:
1720
using: "composite"
1821
steps:
19-
- name: Download artifact
20-
uses: ./.github/actions/artifact_download
21-
with:
22-
profile: release
23-
sha: ${{ github.sha }}
24-
target: x86_64-unknown-linux-gnu
25-
26-
- name: Setup Databend Binary
27-
shell: bash
28-
run: |
29-
sudo cp ./target/release/databend-query /usr/local/bin/databend-query
30-
sudo cp ./target/release/databend-meta /usr/local/bin/databend-meta
31-
databend-query --version
32-
databend-meta --version
33-
34-
- name: Setup BendSQL
35-
shell: bash
36-
run: |
37-
curl -fsSLo ./bendsql.tar.gz https://github.com/databendcloud/bendsql/releases/download/v0.3.5/bendsql-linux-amd64.tar.gz
38-
tar -xvzf ./bendsql.tar.gz
39-
chmod +x ./linux-amd64/bendsql
40-
sudo cp ./linux-amd64/bendsql /usr/local/bin/bendsql
41-
bendsql version
42-
4322
- name: Install script dependencies
4423
shell: bash
4524
run: |
@@ -49,29 +28,16 @@ runs:
4928
- name: Run Benchmark
5029
working-directory: benchmark/clickbench
5130
env:
52-
BENCHMARK_ID: ${{ github.run_id }}
53-
BENCHMARK_STORAGE: ${{ inputs.storage }}
31+
BENCHMARK_ID: ${{ inputs.run_id }}
5432
BENCHMARK_DATASET: ${{ inputs.dataset }}
5533
shell: bash
5634
run: |
57-
./benchmark.sh
35+
./benchmark_local.sh
5836
5937
- name: Prepare Metadata
6038
working-directory: benchmark/clickbench
6139
shell: bash
6240
run: |
63-
case ${{ inputs.storage }} in
64-
s3)
65-
BENCHMARK_STORAGE="s3"
66-
;;
67-
fs)
68-
BENCHMARK_STORAGE="gp3"
69-
;;
70-
*)
71-
echo "Unspported benchmark storage: ${{ inputs.storage }}"
72-
exit 1
73-
esac
74-
7541
case ${{ inputs.source }} in
7642
pr)
7743
BENCHMARK_SYSTEM="Databend(PR#${{ inputs.source_id }})"
@@ -87,18 +53,18 @@ runs:
8753
esac
8854
jq ".system = \"${BENCHMARK_SYSTEM}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
8955
jq ".date = \"$(date -u +%Y-%m-%d)\"" <result.json >result.json.tmp && mv result.json.tmp result.json
90-
jq ".machine = \"c5.4xlarge,${BENCHMARK_STORAGE}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
56+
jq ".machine = \"c5.4xlarge,gp3\"" <result.json >result.json.tmp && mv result.json.tmp result.json
9157
jq ".cluster_size = 1" <result.json >result.json.tmp && mv result.json.tmp result.json
92-
jq ".comment = \"commit:${{ github.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
93-
jq ".tags = [\"${BENCHMARK_TYPE}\", \"${BENCHMARK_STORAGE}\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
58+
jq ".comment = \"commit:${{ inputs.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
59+
jq ".tags = [\"${BENCHMARK_TYPE}\", \"gp3\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
9460
jq ".extra.${{ inputs.source }} = \"${{ inputs.source_id }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
95-
mv result.json result-${{ inputs.dataset }}-${{ inputs.storage }}.json
61+
mv result.json result-${{ inputs.dataset }}-local.json
9662
9763
- name: Upload artifact
9864
uses: actions/upload-artifact@v3
9965
with:
100-
name: benchmark-${{ inputs.dataset }}-${{ inputs.storage }}
101-
path: benchmark/clickbench/result-${{ inputs.dataset }}-${{ inputs.storage }}.json
66+
name: benchmark-${{ inputs.dataset }}-local
67+
path: benchmark/clickbench/result-${{ inputs.dataset }}-local.json
10268

10369
- name: Clean test data
10470
working-directory: benchmark/clickbench

‎.github/actions/publish_binary/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ runs:
3737
GH_TOKEN: ${{ inputs.github_token }}
3838

3939
- name: Configure AWS credentials
40-
uses: aws-actions/configure-aws-credentials@v1
40+
uses: aws-actions/configure-aws-credentials@v2
4141
with:
4242
role-duration-seconds: 900
4343
role-to-assume: ${{ inputs.repo_role_arn }}

‎.github/actions/publish_deb/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ runs:
4141
reprepro includedeb stable databend_${deb_version}_arm64.deb
4242
4343
- name: Configure AWS credentials
44-
uses: aws-actions/configure-aws-credentials@v1
44+
uses: aws-actions/configure-aws-credentials@v2
4545
with:
4646
role-duration-seconds: 900
4747
role-to-assume: ${{ inputs.repo_role_arn }}

‎.github/actions/setup_bendsql/action.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: "Setup BendSQL"
2+
description: "Setup BendSQL for CI"
3+
inputs:
4+
version:
5+
description: "bendsql version"
6+
required: false
7+
default: v0.3.7
8+
runs:
9+
using: "composite"
10+
steps:
11+
- name: Download and Install
12+
shell: bash
13+
run: |
14+
curl -fsSLo ./bendsql.tar.gz https://github.com/databendcloud/bendsql/releases/download/${{ inputs.version }}/bendsql-linux-amd64.tar.gz
15+
tar -xvzf ./bendsql.tar.gz
16+
chmod +x ./linux-amd64/bendsql
17+
sudo cp ./linux-amd64/bendsql /usr/local/bin/bendsql
18+
bendsql version

‎.github/actions/setup_docker/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ runs:
2929

3030
- name: Configure AWS Credentials
3131
if: inputs.ecr_role_arn
32-
uses: aws-actions/configure-aws-credentials@v1
32+
uses: aws-actions/configure-aws-credentials@v2
3333
with:
3434
role-to-assume: ${{ inputs.ecr_role_arn }}
3535
role-duration-seconds: 900

‎.github/actions/test_sqllogic_standalone_linux/action.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,8 @@ runs:
3131
env:
3232
TEST_HANDLERS: ${{ inputs.handlers }}
3333
run: bash ./scripts/ci/ci-run-sqllogic-tests.sh ${{ inputs.dirs }}
34+
- name: Run native sqllogic Tests with Standalone mode
35+
shell: bash
36+
env:
37+
TEST_HANDLERS: ${{ inputs.handlers }}
38+
run: bash ./scripts/ci/ci-run-sqllogic-tests-native.sh

‎.github/workflows/benchmark.yml‎

Lines changed: 10 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,6 @@ on:
1515
- "**.md"
1616
- "docker/**"
1717
- ".devcontainer/**"
18-
workflow_dispatch:
19-
inputs:
20-
tag:
21-
description: The tag to run benchmark
22-
required: true
23-
type: string
2418

2519
concurrency:
2620
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
@@ -29,13 +23,10 @@ concurrency:
2923
env:
3024
BUILD_PROFILE: release
3125
RUNNER_PROVIDER: aws
32-
CARGO_INCREMENTAL: 0
33-
# Remove this env after rust 1.68 has been released.
34-
CARGO_UNSTABLE_SPARSE_REGISTRY: true
3526

3627
jobs:
3728
build_release:
38-
if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ci-benchmark')
29+
if: contains(github.event.pull_request.labels.*.name, 'ci-benchmark')
3930
timeout-minutes: 30
4031
runs-on: [self-hosted, X64, Linux, perf]
4132
steps:
@@ -47,40 +38,14 @@ jobs:
4738
with:
4839
target: x86_64-unknown-linux-gnu
4940
profile: release
50-
51-
clickbench:
52-
needs: build_release
53-
timeout-minutes: 30
54-
runs-on: [self-hosted, X64, Linux, perf]
55-
strategy:
56-
matrix:
57-
storage:
58-
- s3
59-
- fs
60-
dataset:
61-
- hits
62-
- tpch
63-
steps:
64-
- uses: actions/checkout@v3
65-
- name: Get benchmark source
41+
- name: Generate benchmark metadata
6642
run: |
67-
case ${{ github.event_name }} in
68-
pull_request)
69-
echo "BENCHMARK_SOURCE=pr" > $GITHUB_ENV
70-
echo "BENCHMARK_SOURCE_ID=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
71-
;;
72-
workflow_dispatch)
73-
echo "BENCHMARK_SOURCE=release" > $GITHUB_ENV
74-
echo "BENCHMARK_SOURCE_ID=${{ github.event.inputs.tag }}" >> $GITHUB_ENV
75-
;;
76-
*)
77-
echo "Unknown event name: ${{ github.event_name }}"
78-
exit 1
79-
;;
80-
esac
81-
- uses: ./.github/actions/benchmark
43+
echo '{}' > metadata.json
44+
jq ".sha = \"${{ github.sha }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
45+
jq ".run_id = \"${{ github.run_id }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
46+
jq ".pr_id = \"${{ github.event.pull_request.number }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
47+
- name: Upload artifact
48+
uses: actions/upload-artifact@v3
8249
with:
83-
storage: ${{ matrix.storage }}
84-
dataset: ${{ matrix.dataset }}
85-
source: ${{ env.BENCHMARK_SOURCE }}
86-
source_id: ${{ env.BENCHMARK_SOURCE_ID }}
50+
name: benchmark-metadata
51+
path: metadata.json

‎.github/workflows/dev-macos.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,5 +65,5 @@ jobs:
6565
# runs-on: macos-11
6666
# needs: build_macos
6767
# steps:
68-
# - uses: actions/checkout@v2
68+
# - uses: actions/checkout@v3
6969
# - uses: ./.github/actions/test_sqllogic_standalone_macos

‎.github/workflows/production.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ jobs:
153153
- "base"
154154
- "ydb"
155155
steps:
156-
- uses: actions/checkout@v2
156+
- uses: actions/checkout@v3
157157
- uses: ./.github/actions/test_sqllogic_standalone_macos
158158
with:
159159
profile: release
@@ -177,7 +177,7 @@ jobs:
177177
- "http"
178178
- "clickhouse"
179179
steps:
180-
- uses: actions/checkout@v2
180+
- uses: actions/checkout@v3
181181
- uses: ./.github/actions/test_sqllogic_standalone_linux
182182
with:
183183
profile: release

0 commit comments

Comments
 (0)