Skip to content

Commit 1602ec8

Browse files
maxknvmaxknv
authored
Merge pull request ClickHouse#78657 from ClickHouse/ci_fix_nightly
CI: Refactor Nightly workflows
2 parents b0e5ee6 + e6b34d0 commit 1602ec8

File tree

10 files changed

+434
-212
lines changed

10 files changed

+434
-212
lines changed

‎.github/workflows/master.yml‎

Lines changed: 185 additions & 109 deletions
Large diffs are not rendered by default.

‎.github/workflows/nightly.yml‎

Lines changed: 0 additions & 91 deletions
This file was deleted.

‎.github/workflows/nightly_fuzzers.yml‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
# generated by praktika
2+
3+
name: NightlyFuzzers
4+
on:
5+
schedule:
6+
- cron: 13 3 * * *
7+
workflow_dispatch:
8+
9+
concurrency:
10+
group: ${{ github.workflow }}
11+
12+
env:
13+
PYTHONUNBUFFERED: 1
14+
CHECKOUT_REF: ""
15+
16+
jobs:
17+
18+
config_workflow:
19+
runs-on: [self-hosted, style-checker-aarch64]
20+
needs: []
21+
name: "Config Workflow"
22+
outputs:
23+
data: ${{ steps.run.outputs.DATA }}
24+
steps:
25+
- name: Checkout code
26+
uses: actions/checkout@v4
27+
with:
28+
ref: ${{ env.CHECKOUT_REF }}
29+
30+
- name: Prepare env script
31+
run: |
32+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
33+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
34+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
35+
export PYTHONPATH=./ci:.:
36+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
37+
${{ needs.config_workflow.outputs.data }}
38+
EOF
39+
cat > ./ci/tmp/workflow_status.json << 'EOF'
40+
${{ toJson(needs) }}
41+
EOF
42+
ENV_SETUP_SCRIPT_EOF
43+
44+
- name: Run
45+
id: run
46+
run: |
47+
. ./ci/tmp/praktika_setup_env.sh
48+
set -o pipefail
49+
if command -v ts &> /dev/null; then
50+
python3 -m praktika run 'Config Workflow' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
51+
else
52+
python3 -m praktika run 'Config Workflow' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
53+
fi
54+
55+
dockers_build_arm:
56+
runs-on: [self-hosted, style-checker-aarch64]
57+
needs: [config_workflow]
58+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYXJtKQ==') }}
59+
name: "Dockers Build (arm)"
60+
outputs:
61+
data: ${{ steps.run.outputs.DATA }}
62+
steps:
63+
- name: Checkout code
64+
uses: actions/checkout@v4
65+
with:
66+
ref: ${{ env.CHECKOUT_REF }}
67+
68+
- name: Prepare env script
69+
run: |
70+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
71+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
72+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
73+
export PYTHONPATH=./ci:.:
74+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
75+
${{ needs.config_workflow.outputs.data }}
76+
EOF
77+
cat > ./ci/tmp/workflow_status.json << 'EOF'
78+
${{ toJson(needs) }}
79+
EOF
80+
ENV_SETUP_SCRIPT_EOF
81+
82+
- name: Run
83+
id: run
84+
run: |
85+
. ./ci/tmp/praktika_setup_env.sh
86+
set -o pipefail
87+
if command -v ts &> /dev/null; then
88+
python3 -m praktika run 'Dockers Build (arm)' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
89+
else
90+
python3 -m praktika run 'Dockers Build (arm)' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
91+
fi
92+
93+
dockers_build_amd_and_merge:
94+
runs-on: [self-hosted, style-checker]
95+
needs: [config_workflow, dockers_build_arm]
96+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYW1kKSBhbmQgTWVyZ2U=') }}
97+
name: "Dockers Build (amd) and Merge"
98+
outputs:
99+
data: ${{ steps.run.outputs.DATA }}
100+
steps:
101+
- name: Checkout code
102+
uses: actions/checkout@v4
103+
with:
104+
ref: ${{ env.CHECKOUT_REF }}
105+
106+
- name: Prepare env script
107+
run: |
108+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
109+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
110+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
111+
export PYTHONPATH=./ci:.:
112+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
113+
${{ needs.config_workflow.outputs.data }}
114+
EOF
115+
cat > ./ci/tmp/workflow_status.json << 'EOF'
116+
${{ toJson(needs) }}
117+
EOF
118+
ENV_SETUP_SCRIPT_EOF
119+
120+
- name: Run
121+
id: run
122+
run: |
123+
. ./ci/tmp/praktika_setup_env.sh
124+
set -o pipefail
125+
if command -v ts &> /dev/null; then
126+
python3 -m praktika run 'Dockers Build (amd) and Merge' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
127+
else
128+
python3 -m praktika run 'Dockers Build (amd) and Merge' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
129+
fi
130+
131+
build_fuzzers:
132+
runs-on: [self-hosted, builder-aarch64]
133+
needs: [config_workflow, dockers_build_amd_and_merge]
134+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGZ1enplcnMp') }}
135+
name: "Build (fuzzers)"
136+
outputs:
137+
data: ${{ steps.run.outputs.DATA }}
138+
steps:
139+
- name: Checkout code
140+
uses: actions/checkout@v4
141+
with:
142+
ref: ${{ env.CHECKOUT_REF }}
143+
144+
- name: Prepare env script
145+
run: |
146+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
147+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
148+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
149+
export PYTHONPATH=./ci:.:
150+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
151+
${{ needs.config_workflow.outputs.data }}
152+
EOF
153+
cat > ./ci/tmp/workflow_status.json << 'EOF'
154+
${{ toJson(needs) }}
155+
EOF
156+
ENV_SETUP_SCRIPT_EOF
157+
158+
- name: Run
159+
id: run
160+
run: |
161+
. ./ci/tmp/praktika_setup_env.sh
162+
set -o pipefail
163+
if command -v ts &> /dev/null; then
164+
python3 -m praktika run 'Build (fuzzers)' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
165+
else
166+
python3 -m praktika run 'Build (fuzzers)' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
167+
fi
168+
169+
libfuzzer_tests:
170+
runs-on: [self-hosted, func-tester]
171+
needs: [config_workflow, dockers_build_amd_and_merge, build_fuzzers]
172+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'bGliRnV6emVyIHRlc3Rz') }}
173+
name: "libFuzzer tests"
174+
outputs:
175+
data: ${{ steps.run.outputs.DATA }}
176+
steps:
177+
- name: Checkout code
178+
uses: actions/checkout@v4
179+
with:
180+
ref: ${{ env.CHECKOUT_REF }}
181+
182+
- name: Prepare env script
183+
run: |
184+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
185+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
186+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
187+
export PYTHONPATH=./ci:.:
188+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
189+
${{ needs.config_workflow.outputs.data }}
190+
EOF
191+
cat > ./ci/tmp/workflow_status.json << 'EOF'
192+
${{ toJson(needs) }}
193+
EOF
194+
ENV_SETUP_SCRIPT_EOF
195+
196+
- name: Run
197+
id: run
198+
run: |
199+
. ./ci/tmp/praktika_setup_env.sh
200+
set -o pipefail
201+
if command -v ts &> /dev/null; then
202+
python3 -m praktika run 'libFuzzer tests' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
203+
else
204+
python3 -m praktika run 'libFuzzer tests' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
205+
fi

‎ci/defs/job_configs.py‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -913,28 +913,22 @@ class JobConfigs:
913913
bugfix_validation = Job.Config(
914914
name=JobNames.BUGFIX_VALIDATE,
915915
runs_on=RunnerLabels.STYLE_CHECK_AMD,
916-
digest_config=Job.CacheDigestConfig(
917-
include_paths=["./tests/ci/sqltest.py"],
918-
),
919916
command="cd ./tests/ci && python3 ci.py --run-from-praktika",
920917
requires=["Build (amd_debug)"],
921918
)
922-
# TODO: run by labels Labels.JEPSEN_TEST
923919
jepsen_keeper = Job.Config(
924920
name=JobNames.JEPSEN_KEEPER,
925921
runs_on=RunnerLabels.STYLE_CHECK_AMD,
926922
command="cd ./tests/ci && python3 ci.py --run-from-praktika",
927923
requires=["Build (amd_binary)"],
928924
)
929-
# TODO: run by labels Labels.JEPSEN_TEST
930925
jepsen_server = Job.Config(
931926
name=JobNames.JEPSEN_KEEPER,
932927
runs_on=RunnerLabels.STYLE_CHECK_AMD,
933928
command="cd ./tests/ci && python3 ci.py --run-from-praktika",
934929
requires=["Build (amd_binary)"],
935930
)
936-
# TODO: run by label Tags.libFuzzer
937-
libfuzzer_jjob = Job.Config(
931+
libfuzzer_job = Job.Config(
938932
name=JobNames.LIBFUZZER_TEST,
939933
runs_on=RunnerLabels.FUNC_TESTER_AMD,
940934
command="cd ./tests/ci && python3 ci.py --run-from-praktika",

‎ci/jobs/scripts/workflow_hooks/filter_job.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from praktika.info import Info
1+
from ci.praktika.info import Info
22

33
from ci.defs.defs import JobNames
44
from ci.jobs.scripts.workflow_hooks.pr_description import Labels
@@ -70,6 +70,9 @@ def should_skip_job(job_name):
7070
"Skipped, labeled with 'ci-performance' - run performance jobs only",
7171
)
7272

73+
if "- Bug Fix" not in _info_cache.pr_body and JobNames.BUGFIX_VALIDATE in job_name:
74+
return True, "Skipped, not a bug-fix PR"
75+
7376
# skip ARM perf tests for non-performance update
7477
if (
7578
# Labels.PR_PERFORMANCE not in _info_cache.pr_labels

‎ci/praktika/docker.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ class Config:
1919

2020
@classmethod
2121
def build(cls, config: "Docker.Config", digests, amd_only, arm_only, with_log):
22-
from praktika.result import Result
22+
from .result import Result
2323

2424
sw = Utils.Stopwatch()
2525
tag = digests[config.name]
@@ -75,7 +75,7 @@ def merge_manifest(
7575
cls, config: "Docker.Config", digests, add_latest, with_log=False
7676
):
7777

78-
from praktika.result import Result
78+
from .result import Result
7979

8080
tags = [digests[config.name]]
8181

‎ci/praktika/native_jobs.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ def _build_dockers(workflow, job_name):
165165
config=docker,
166166
digests=docker_digests,
167167
with_log=True,
168-
add_latest=False,
168+
add_latest=workflow.set_latest_in_dockers_build,
169169
)
170170
)
171171

‎ci/praktika/workflow.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ class Config:
4444
# If the Docker images specified in .dockers are intended to be built in a different workflow,
4545
# their build process in this workflow can be disabled by setting this to True.
4646
disable_dockers_build: bool = False
47+
set_latest_in_dockers_build: bool = False
4748

4849
def is_event_pull_request(self):
4950
return self.event == Workflow.Event.PULL_REQUEST

‎ci/workflows/master.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@
5555
*ArtifactConfigs.performance_reports,
5656
],
5757
dockers=DOCKERS,
58-
disable_dockers_build=True,
58+
set_latest_in_dockers_build=True,
5959
secrets=SECRETS,
6060
enable_cache=True,
6161
enable_report=True,

0 commit comments

Comments
 (0)