Skip to content

Commit 02e208d

Browse files
strtgbbstrtgbb
committed
cherry-pick #1054 
1 parent 6fff60c commit 02e208d

File tree

3 files changed

+42
-47
lines changed

3 files changed

+42
-47
lines changed

‎.github/workflows/release_builds.yml‎

Lines changed: 5 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,6 @@ jobs:
8989
dockers_build_amd:
9090
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
9191
needs: [config_workflow]
92-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYW1kKQ==') }}
9392
name: "Dockers Build (amd)"
9493
outputs:
9594
data: ${{ steps.run.outputs.DATA }}
@@ -137,7 +136,6 @@ jobs:
137136
dockers_build_arm:
138137
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
139138
needs: [config_workflow]
140-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYXJtKQ==') }}
141139
name: "Dockers Build (arm)"
142140
outputs:
143141
data: ${{ steps.run.outputs.DATA }}
@@ -184,8 +182,7 @@ jobs:
184182
185183
build_amd_debug:
186184
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
187-
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
188-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9kZWJ1Zyk=') }}
185+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary, build_arm_binary]
189186
name: "Build (amd_debug)"
190187
outputs:
191188
data: ${{ steps.run.outputs.DATA }}
@@ -233,7 +230,6 @@ jobs:
233230
build_amd_release:
234231
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
235232
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
236-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
237233
name: "Build (amd_release)"
238234
outputs:
239235
data: ${{ steps.run.outputs.DATA }}
@@ -280,8 +276,7 @@ jobs:
280276
281277
build_amd_asan:
282278
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
283-
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
284-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9hc2FuKQ==') }}
279+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary, build_arm_binary]
285280
name: "Build (amd_asan)"
286281
outputs:
287282
data: ${{ steps.run.outputs.DATA }}
@@ -328,8 +323,7 @@ jobs:
328323
329324
build_amd_tsan:
330325
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
331-
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
332-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF90c2FuKQ==') }}
326+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary, build_arm_binary]
333327
name: "Build (amd_tsan)"
334328
outputs:
335329
data: ${{ steps.run.outputs.DATA }}
@@ -376,8 +370,7 @@ jobs:
376370
377371
build_amd_msan:
378372
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
379-
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
380-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9tc2FuKQ==') }}
373+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary, build_arm_binary]
381374
name: "Build (amd_msan)"
382375
outputs:
383376
data: ${{ steps.run.outputs.DATA }}
@@ -424,8 +417,7 @@ jobs:
424417
425418
build_amd_ubsan:
426419
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
427-
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
428-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF91YnNhbik=') }}
420+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary, build_arm_binary]
429421
name: "Build (amd_ubsan)"
430422
outputs:
431423
data: ${{ steps.run.outputs.DATA }}
@@ -473,7 +465,6 @@ jobs:
473465
build_amd_binary:
474466
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
475467
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
476-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9iaW5hcnkp') }}
477468
name: "Build (amd_binary)"
478469
outputs:
479470
data: ${{ steps.run.outputs.DATA }}
@@ -521,7 +512,6 @@ jobs:
521512
build_arm_release:
522513
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
523514
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
524-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
525515
name: "Build (arm_release)"
526516
outputs:
527517
data: ${{ steps.run.outputs.DATA }}
@@ -569,7 +559,6 @@ jobs:
569559
build_arm_binary:
570560
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
571561
needs: [config_workflow, dockers_build_amd, dockers_build_arm]
572-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9iaW5hcnkp') }}
573562
name: "Build (arm_binary)"
574563
outputs:
575564
data: ${{ steps.run.outputs.DATA }}
@@ -617,7 +606,6 @@ jobs:
617606
docker_server_image:
618607
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
619608
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_release, build_arm_release]
620-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIHNlcnZlciBpbWFnZQ==') }}
621609
name: "Docker server image"
622610
outputs:
623611
data: ${{ steps.run.outputs.DATA }}
@@ -665,7 +653,6 @@ jobs:
665653
docker_keeper_image:
666654
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
667655
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_release, build_arm_release]
668-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
669656
name: "Docker keeper image"
670657
outputs:
671658
data: ${{ steps.run.outputs.DATA }}
@@ -713,7 +700,6 @@ jobs:
713700
install_packages_amd_release:
714701
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester]
715702
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_release]
716-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'SW5zdGFsbCBwYWNrYWdlcyAoYW1kX3JlbGVhc2Up') }}
717703
name: "Install packages (amd_release)"
718704
outputs:
719705
data: ${{ steps.run.outputs.DATA }}
@@ -761,7 +747,6 @@ jobs:
761747
install_packages_arm_release:
762748
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
763749
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_arm_release]
764-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'SW5zdGFsbCBwYWNrYWdlcyAoYXJtX3JlbGVhc2Up') }}
765750
name: "Install packages (arm_release)"
766751
outputs:
767752
data: ${{ steps.run.outputs.DATA }}
@@ -809,7 +794,6 @@ jobs:
809794
stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_parallel:
810795
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester]
811796
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary]
812-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBvbGQgYW5hbHl6ZXIsIHMzIHN0b3JhZ2UsIERhdGFiYXNlUmVwbGljYXRlZCwgcGFyYWxsZWwp') }}
813797
name: "Stateless tests (amd_binary, old analyzer, s3 storage, DatabaseReplicated, parallel)"
814798
outputs:
815799
data: ${{ steps.run.outputs.DATA }}
@@ -857,7 +841,6 @@ jobs:
857841
stateless_tests_amd_binary_old_analyzer_s3_storage_databasereplicated_sequential:
858842
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester]
859843
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary]
860-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBvbGQgYW5hbHl6ZXIsIHMzIHN0b3JhZ2UsIERhdGFiYXNlUmVwbGljYXRlZCwgc2VxdWVudGlhbCk=') }}
861844
name: "Stateless tests (amd_binary, old analyzer, s3 storage, DatabaseReplicated, sequential)"
862845
outputs:
863846
data: ${{ steps.run.outputs.DATA }}
@@ -905,7 +888,6 @@ jobs:
905888
stateless_tests_amd_binary_parallelreplicas_s3_storage_parallel:
906889
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester]
907890
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary]
908-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBQYXJhbGxlbFJlcGxpY2FzLCBzMyBzdG9yYWdlLCBwYXJhbGxlbCk=') }}
909891
name: "Stateless tests (amd_binary, ParallelReplicas, s3 storage, parallel)"
910892
outputs:
911893
data: ${{ steps.run.outputs.DATA }}
@@ -953,7 +935,6 @@ jobs:
953935
stateless_tests_amd_binary_parallelreplicas_s3_storage_sequential:
954936
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester]
955937
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_amd_binary]
956-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBQYXJhbGxlbFJlcGxpY2FzLCBzMyBzdG9yYWdlLCBzZXF1ZW50aWFsKQ==') }}
957938
name: "Stateless tests (amd_binary, ParallelReplicas, s3 storage, sequential)"
958939
outputs:
959940
data: ${{ steps.run.outputs.DATA }}
@@ -1001,7 +982,6 @@ jobs:
1001982
stateless_tests_arm_binary_parallel:
1002983
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
1003984
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_arm_binary]
1004-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYmluYXJ5LCBwYXJhbGxlbCk=') }}
1005985
name: "Stateless tests (arm_binary, parallel)"
1006986
outputs:
1007987
data: ${{ steps.run.outputs.DATA }}
@@ -1049,7 +1029,6 @@ jobs:
10491029
stateless_tests_arm_binary_sequential:
10501030
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
10511031
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_arm_binary]
1052-
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYmluYXJ5LCBzZXF1ZW50aWFsKQ==') }}
10531032
name: "Stateless tests (arm_binary, sequential)"
10541033
outputs:
10551034
data: ${{ steps.run.outputs.DATA }}

‎ci/jobs/scripts/workflow_hooks/version_log.py‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,24 @@
88

99

1010
def _add_build_to_version_history():
11-
info = Info()
12-
Shell.check(
13-
f"git rev-parse --is-shallow-repository | grep -q true && git fetch --unshallow --prune --no-recurse-submodules --filter=tree:0 origin {info.git_branch} ||:"
14-
)
15-
commit_parents = Shell.get_output("git log --format=%P -n 1").split(" ")
16-
data = {
17-
"check_start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
18-
"pull_request_number": info.pr_number,
19-
"pull_request_url": info.pr_url,
20-
"commit_sha": info.sha,
21-
"commit_url": info.commit_url,
22-
"parent_commits_sha": commit_parents,
23-
"version": CHVersion.get_version(),
24-
"git_ref": info.git_branch,
25-
}
26-
print(f"Update version log: [{data}]")
27-
CIDBCluster().insert_json(table="version_history", json_str=data)
11+
# info = Info()
12+
# Shell.check(
13+
# f"git rev-parse --is-shallow-repository | grep -q true && git fetch --unshallow --prune --no-recurse-submodules --filter=tree:0 origin {info.git_branch} ||:"
14+
# )
15+
# commit_parents = Shell.get_output("git log --format=%P -n 1").split(" ")
16+
# data = {
17+
# "check_start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
18+
# "pull_request_number": info.pr_number,
19+
# "pull_request_url": info.pr_url,
20+
# "commit_sha": info.sha,
21+
# "commit_url": info.commit_url,
22+
# "parent_commits_sha": commit_parents,
23+
# "version": CHVersion.get_version(),
24+
# "git_ref": info.git_branch,
25+
# }
26+
# print(f"Update version log: [{data}]")
27+
# CIDBCluster().insert_json(table="version_history", json_str=data)
28+
2829
# stores actual version data in pipline storage, to be used by jobs that need it
2930
CHVersion.store_version_data_in_ci_pipeline()
3031

‎ci/workflows/release_builds.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,26 @@
1010
if "coverage" not in job.name
1111
]
1212

13+
# Make sure that builds that get tested are built first
14+
# Note that the release build job should not block or be blocked, it is long and it's dependencies are fast.
15+
BLOCKING_BUILD_JOBS = [
16+
job.name
17+
for job in JobConfigs.build_jobs
18+
if any(substr in job.name for substr in ["binary"])
19+
]
20+
1321
workflow = Workflow.Config(
1422
name="Release Builds",
1523
event=Workflow.Event.DISPATCH,
1624
jobs=[
17-
*builds_for_release_branch,
25+
*[
26+
job.set_dependency(
27+
BLOCKING_BUILD_JOBS
28+
if job.name not in BLOCKING_BUILD_JOBS and "release" not in job.name
29+
else []
30+
)
31+
for job in builds_for_release_branch
32+
],
1833
JobConfigs.docker_sever,
1934
JobConfigs.docker_keeper,
2035
*JobConfigs.install_check_master_jobs,
@@ -34,13 +49,13 @@
3449
],
3550
dockers=DOCKERS,
3651
secrets=SECRETS,
37-
enable_job_filtering_by_changes=True,
38-
enable_cache=True,
52+
enable_job_filtering_by_changes=False,
53+
enable_cache=False,
3954
enable_report=True,
4055
enable_cidb=True,
4156
enable_commit_status_on_failure=True,
4257
pre_hooks=[
43-
"python3 ./ci/jobs/scripts/workflow_hooks/store_data.py",
58+
# "python3 ./ci/jobs/scripts/workflow_hooks/store_data.py",
4459
"python3 ./ci/jobs/scripts/workflow_hooks/version_log.py",
4560
],
4661
workflow_filter_hooks=[should_skip_job],

0 commit comments

Comments
 (0)