Skip to content

Commit 097799b

Browse files
taiyang-litaiyang-li
committed
Merge branch 'opt_count_if' of https://github.com/bigo-sg/ClickHouse into opt_count_if
2 parents b822e54 + ac6a99c commit 097799b

File tree

3,642 files changed

+73993
-56687
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

3,642 files changed

+73993
-56687
lines changed

‎.github/workflows/backport_branches.yml‎

Lines changed: 1174 additions & 259 deletions
Large diffs are not rendered by default.

‎.github/workflows/master.yml‎

Lines changed: 452 additions & 377 deletions
Large diffs are not rendered by default.

‎.github/workflows/merge_queue.yml‎

Lines changed: 256 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -1,114 +1,279 @@
1-
# yamllint disable rule:comments-indentation
1+
# generated by praktika
2+
23
name: MergeQueueCI
34

5+
on:
6+
merge_group:
7+
48
env:
59
# Force the stdout and stderr streams to be unbuffered
610
PYTHONUNBUFFERED: 1
11+
CHECKOUT_REF: ""
712

8-
on: # yamllint disable-line rule:truthy
9-
merge_group:
1013

1114
jobs:
12-
RunConfig:
15+
16+
config_workflow:
1317
runs-on: [self-hosted, style-checker-aarch64]
18+
needs: []
19+
name: "Config Workflow"
1420
outputs:
15-
data: ${{ steps.runconfig.outputs.CI_DATA }}
21+
data: ${{ steps.run.outputs.DATA }}
1622
steps:
17-
- name: Check out repository code
18-
uses: ClickHouse/checkout@v1
23+
- name: Checkout code
24+
uses: actions/checkout@v4
1925
with:
20-
clear-repository: true # to ensure correct digests
21-
fetch-depth: 0 # to get a version
22-
filter: tree:0
23-
- name: Debug Info
24-
uses: ./.github/actions/debug
25-
- name: Cancel PR workflow
26+
ref: ${{ env.CHECKOUT_REF }}
27+
28+
- name: Prepare env script
2629
run: |
27-
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
28-
- name: Python unit tests
30+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
31+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
32+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
33+
export PYTHONPATH=./ci:.:
34+
cat > ./ci/tmp/workflow_config_mergequeueci.json << 'EOF'
35+
${{ needs.config_workflow.outputs.data }}
36+
EOF
37+
cat > ./ci/tmp/workflow_status.json << 'EOF'
38+
${{ toJson(needs) }}
39+
EOF
40+
ENV_SETUP_SCRIPT_EOF
41+
42+
- name: Run
43+
id: run
2944
run: |
30-
cd "$GITHUB_WORKSPACE/tests/ci"
31-
echo "Testing the main ci directory"
32-
python3 -m unittest discover -s . -p 'test_*.py'
33-
- name: PrepareRunConfig
34-
id: runconfig
45+
. ./ci/tmp/praktika_setup_env.sh
46+
set -o pipefail
47+
if command -v ts &> /dev/null; then
48+
python3 -m praktika run 'Config Workflow' --workflow "MergeQueueCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
49+
else
50+
python3 -m praktika run 'Config Workflow' --workflow "MergeQueueCI" --ci |& tee ./ci/tmp/job.log
51+
fi
52+
53+
dockers_build_arm:
54+
runs-on: [self-hosted, style-checker-aarch64]
55+
needs: [config_workflow]
56+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYXJtKQ==') }}
57+
name: "Dockers Build (arm)"
58+
outputs:
59+
data: ${{ steps.run.outputs.DATA }}
60+
steps:
61+
- name: Checkout code
62+
uses: actions/checkout@v4
63+
with:
64+
ref: ${{ env.CHECKOUT_REF }}
65+
66+
- name: Prepare env script
3567
run: |
36-
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json
37-
38-
echo "::group::CI configuration"
39-
python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
40-
echo "::endgroup::"
41-
42-
{
43-
echo 'CI_DATA<<EOF'
44-
cat ${{ runner.temp }}/ci_run_data.json
45-
echo 'EOF'
46-
} >> "$GITHUB_OUTPUT"
47-
BuildDockers:
48-
needs: [RunConfig]
49-
if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }}
50-
uses: ./.github/workflows/docker_test_images.yml
51-
with:
52-
data: ${{ needs.RunConfig.outputs.data }}
53-
StyleCheck:
54-
needs: [RunConfig, BuildDockers]
55-
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}}
56-
uses: ./.github/workflows/reusable_test.yml
57-
with:
58-
test_name: Style check
59-
runner_type: style-checker-aarch64
60-
run_command: |
61-
python3 style_check.py --no-push
62-
data: ${{ needs.RunConfig.outputs.data }}
63-
FastTest:
64-
needs: [RunConfig, BuildDockers]
65-
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
66-
uses: ./.github/workflows/reusable_test.yml
67-
with:
68-
test_name: Fast test
69-
runner_type: builder
70-
data: ${{ needs.RunConfig.outputs.data }}
71-
run_command: |
72-
python3 fast_test_check.py
73-
74-
Builds_1:
75-
needs: [RunConfig, BuildDockers]
76-
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
77-
# using callable wf (reusable_stage.yml) allows grouping all nested jobs under a tab
78-
uses: ./.github/workflows/reusable_build_stage.yml
79-
with:
80-
stage: Builds_1
81-
data: ${{ needs.RunConfig.outputs.data }}
82-
Tests_1:
83-
needs: [RunConfig, Builds_1]
84-
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
85-
uses: ./.github/workflows/reusable_test_stage.yml
86-
with:
87-
stage: Tests_1
88-
data: ${{ needs.RunConfig.outputs.data }}
89-
90-
CheckReadyForMerge:
91-
if: ${{ !cancelled() }}
92-
# Test_2 or Test_3 must not have jobs required for Mergeable check
93-
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Tests_1]
68+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
69+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
70+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
71+
export PYTHONPATH=./ci:.:
72+
cat > ./ci/tmp/workflow_config_mergequeueci.json << 'EOF'
73+
${{ needs.config_workflow.outputs.data }}
74+
EOF
75+
cat > ./ci/tmp/workflow_status.json << 'EOF'
76+
${{ toJson(needs) }}
77+
EOF
78+
ENV_SETUP_SCRIPT_EOF
79+
80+
- name: Run
81+
id: run
82+
run: |
83+
. ./ci/tmp/praktika_setup_env.sh
84+
set -o pipefail
85+
if command -v ts &> /dev/null; then
86+
python3 -m praktika run 'Dockers Build (arm)' --workflow "MergeQueueCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
87+
else
88+
python3 -m praktika run 'Dockers Build (arm)' --workflow "MergeQueueCI" --ci |& tee ./ci/tmp/job.log
89+
fi
90+
91+
dockers_build_amd_and_merge:
92+
runs-on: [self-hosted, style-checker]
93+
needs: [config_workflow, dockers_build_arm]
94+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYW1kKSBhbmQgTWVyZ2U=') }}
95+
name: "Dockers Build (amd) and Merge"
96+
outputs:
97+
data: ${{ steps.run.outputs.DATA }}
98+
steps:
99+
- name: Checkout code
100+
uses: actions/checkout@v4
101+
with:
102+
ref: ${{ env.CHECKOUT_REF }}
103+
104+
- name: Prepare env script
105+
run: |
106+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
107+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
108+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
109+
export PYTHONPATH=./ci:.:
110+
cat > ./ci/tmp/workflow_config_mergequeueci.json << 'EOF'
111+
${{ needs.config_workflow.outputs.data }}
112+
EOF
113+
cat > ./ci/tmp/workflow_status.json << 'EOF'
114+
${{ toJson(needs) }}
115+
EOF
116+
ENV_SETUP_SCRIPT_EOF
117+
118+
- name: Run
119+
id: run
120+
run: |
121+
. ./ci/tmp/praktika_setup_env.sh
122+
set -o pipefail
123+
if command -v ts &> /dev/null; then
124+
python3 -m praktika run 'Dockers Build (amd) and Merge' --workflow "MergeQueueCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
125+
else
126+
python3 -m praktika run 'Dockers Build (amd) and Merge' --workflow "MergeQueueCI" --ci |& tee ./ci/tmp/job.log
127+
fi
128+
129+
style_check:
94130
runs-on: [self-hosted, style-checker-aarch64]
131+
needs: [config_workflow, dockers_build_amd_and_merge]
132+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3R5bGUgY2hlY2s=') }}
133+
name: "Style check"
134+
outputs:
135+
data: ${{ steps.run.outputs.DATA }}
136+
steps:
137+
- name: Checkout code
138+
uses: actions/checkout@v4
139+
with:
140+
ref: ${{ env.CHECKOUT_REF }}
141+
142+
- name: Prepare env script
143+
run: |
144+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
145+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
146+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
147+
export PYTHONPATH=./ci:.:
148+
cat > ./ci/tmp/workflow_config_mergequeueci.json << 'EOF'
149+
${{ needs.config_workflow.outputs.data }}
150+
EOF
151+
cat > ./ci/tmp/workflow_status.json << 'EOF'
152+
${{ toJson(needs) }}
153+
EOF
154+
ENV_SETUP_SCRIPT_EOF
155+
156+
- name: Run
157+
id: run
158+
run: |
159+
. ./ci/tmp/praktika_setup_env.sh
160+
set -o pipefail
161+
if command -v ts &> /dev/null; then
162+
python3 -m praktika run 'Style check' --workflow "MergeQueueCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
163+
else
164+
python3 -m praktika run 'Style check' --workflow "MergeQueueCI" --ci |& tee ./ci/tmp/job.log
165+
fi
166+
167+
fast_test:
168+
runs-on: [self-hosted, builder]
169+
needs: [config_workflow, dockers_build_amd_and_merge]
170+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RmFzdCB0ZXN0') }}
171+
name: "Fast test"
172+
outputs:
173+
data: ${{ steps.run.outputs.DATA }}
174+
steps:
175+
- name: Checkout code
176+
uses: actions/checkout@v4
177+
with:
178+
ref: ${{ env.CHECKOUT_REF }}
179+
180+
- name: Prepare env script
181+
run: |
182+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
183+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
184+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
185+
export PYTHONPATH=./ci:.:
186+
cat > ./ci/tmp/workflow_config_mergequeueci.json << 'EOF'
187+
${{ needs.config_workflow.outputs.data }}
188+
EOF
189+
cat > ./ci/tmp/workflow_status.json << 'EOF'
190+
${{ toJson(needs) }}
191+
EOF
192+
ENV_SETUP_SCRIPT_EOF
193+
194+
- name: Run
195+
id: run
196+
run: |
197+
. ./ci/tmp/praktika_setup_env.sh
198+
set -o pipefail
199+
if command -v ts &> /dev/null; then
200+
python3 -m praktika run 'Fast test' --workflow "MergeQueueCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
201+
else
202+
python3 -m praktika run 'Fast test' --workflow "MergeQueueCI" --ci |& tee ./ci/tmp/job.log
203+
fi
204+
205+
build_amd_binary:
206+
runs-on: [self-hosted, builder]
207+
needs: [config_workflow, dockers_build_amd_and_merge]
208+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9iaW5hcnkp') }}
209+
name: "Build (amd_binary)"
210+
outputs:
211+
data: ${{ steps.run.outputs.DATA }}
95212
steps:
96-
- name: Check out repository code
97-
uses: ClickHouse/checkout@v1
98-
- name: Check and set merge status
99-
if: ${{ needs.StyleCheck.result == 'success' }}
213+
- name: Checkout code
214+
uses: actions/checkout@v4
215+
with:
216+
ref: ${{ env.CHECKOUT_REF }}
217+
218+
- name: Prepare env script
100219
run: |
101-
export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
102-
cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
220+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
221+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
222+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
223+
export PYTHONPATH=./ci:.:
224+
cat > ./ci/tmp/workflow_config_mergequeueci.json << 'EOF'
225+
${{ needs.config_workflow.outputs.data }}
226+
EOF
227+
cat > ./ci/tmp/workflow_status.json << 'EOF'
103228
${{ toJson(needs) }}
104229
EOF
105-
cd "$GITHUB_WORKSPACE/tests/ci"
106-
python3 merge_pr.py --set-ci-status
107-
- name: Check Workflow results
108-
if: ${{ !cancelled() }}
230+
ENV_SETUP_SCRIPT_EOF
231+
232+
- name: Run
233+
id: run
234+
run: |
235+
. ./ci/tmp/praktika_setup_env.sh
236+
set -o pipefail
237+
if command -v ts &> /dev/null; then
238+
python3 -m praktika run 'Build (amd_binary)' --workflow "MergeQueueCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
239+
else
240+
python3 -m praktika run 'Build (amd_binary)' --workflow "MergeQueueCI" --ci |& tee ./ci/tmp/job.log
241+
fi
242+
243+
finish_workflow:
244+
runs-on: [self-hosted, style-checker-aarch64]
245+
needs: [config_workflow, dockers_build_arm, dockers_build_amd_and_merge, style_check, fast_test, build_amd_binary]
246+
if: ${{ !cancelled() }}
247+
name: "Finish Workflow"
248+
outputs:
249+
data: ${{ steps.run.outputs.DATA }}
250+
steps:
251+
- name: Checkout code
252+
uses: actions/checkout@v4
253+
with:
254+
ref: ${{ env.CHECKOUT_REF }}
255+
256+
- name: Prepare env script
109257
run: |
110-
export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
111-
cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
258+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
259+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
260+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
261+
export PYTHONPATH=./ci:.:
262+
cat > ./ci/tmp/workflow_config_mergequeueci.json << 'EOF'
263+
${{ needs.config_workflow.outputs.data }}
264+
EOF
265+
cat > ./ci/tmp/workflow_status.json << 'EOF'
112266
${{ toJson(needs) }}
113267
EOF
114-
python3 ./tests/ci/ci_buddy.py --check-wf-status
268+
ENV_SETUP_SCRIPT_EOF
269+
270+
- name: Run
271+
id: run
272+
run: |
273+
. ./ci/tmp/praktika_setup_env.sh
274+
set -o pipefail
275+
if command -v ts &> /dev/null; then
276+
python3 -m praktika run 'Finish Workflow' --workflow "MergeQueueCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
277+
else
278+
python3 -m praktika run 'Finish Workflow' --workflow "MergeQueueCI" --ci |& tee ./ci/tmp/job.log
279+
fi

0 commit comments

Comments
 (0)