Merge branch 'antalya-25.6.5' into simple_export_part

Author: arthurpassos

.github/actions/create_workflow_report/create_workflow_report.py

@@ -762,7 +762,7 @@ def create_workflow_report(
         "date": f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC",
         "is_preview": mark_preview,
         "counts": {
-            "jobs_status": f"{sum(fail_results['job_statuses']['job_status'] != 'success')} fail/error",
+            "jobs_status": f"{sum(fail_results['job_statuses']['job_status'].value_counts().get(x, 0) for x in ('failure', 'error'))} fail/error",
             "checks_errors": len(fail_results["checks_errors"]),
             "checks_new_fails": len(fail_results["checks_fails"]),
             "regression_new_fails": len(fail_results["regression_fails"]),

.github/workflows/backport_branches.yml

@@ -4,14 +4,20 @@ name: BackportPR
 
 on:
   workflow_dispatch:
+    inputs:
+      no_cache:
+        description: Run without cache
+        required: false
+        type: boolean
+        default: false
   pull_request:
     branches: ['2[1-9].[1-9][0-9]', '2[1-9].[1-9]']
 
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
   DISABLE_CI_MERGE_COMMIT: ${{ vars.DISABLE_CI_MERGE_COMMIT || '0' }}
-  DISABLE_CI_CACHE: ${{ vars.DISABLE_CI_CACHE || '0' }}
+  DISABLE_CI_CACHE: ${{ github.event.inputs.no_cache || '0' }}
   CHECKOUT_REF: ${{ vars.DISABLE_CI_MERGE_COMMIT == '1' && github.event.pull_request.head.sha || '' }}
   DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
   CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
@@ -265,6 +271,12 @@ jobs:
             python3 -m praktika run 'Build (amd_release)' --workflow "BackportPR" --ci |& tee ./ci/tmp/job.log
           fi
 
+      - name: Upload artifact build_report_package_release
+        uses: actions/upload-artifact@v4
+        with:
+          name: build_report_package_release
+          path: ./ci/tmp/artifact_report_build_amd_release.json
+
   build_amd_asan:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
     needs: [config_workflow, dockers_build_amd, dockers_build_arm]
@@ -1512,7 +1524,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester
-      commit: aa4204a74b901a0f2ea7c9f1d631d98221554fb1
+      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
       arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300
@@ -1524,7 +1536,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: aa4204a74b901a0f2ea7c9f1d631d98221554fb1
+      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
       arch: aarch64
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300

.github/workflows/master.yml

@@ -4,12 +4,19 @@ name: MasterCI
 
 on:
   workflow_dispatch:
+    inputs:
+      no_cache:
+        description: Run without cache
+        required: false
+        type: boolean
+        default: false
   push:
     branches: ['antalya', 'releases/*', 'antalya-*']
 
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
+  DISABLE_CI_CACHE: ${{ github.event.inputs.no_cache || '0' }}
   CHECKOUT_REF: ""
   DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
   CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
@@ -308,6 +315,12 @@ jobs:
             python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
           fi
 
+      - name: Upload artifact build_report_package_release
+        uses: actions/upload-artifact@v4
+        with:
+          name: build_report_package_release
+          path: ./ci/tmp/artifact_report_build_amd_release.json
+
   build_amd_asan:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
     needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -3985,7 +3998,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester
-      commit: aa4204a74b901a0f2ea7c9f1d631d98221554fb1
+      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
       arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300
@@ -3997,7 +4010,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: aa4204a74b901a0f2ea7c9f1d631d98221554fb1
+      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
       arch: aarch64
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300

.github/workflows/merge_queue.yml

@@ -348,101 +348,3 @@ jobs:
           else
             python3 -m praktika run 'Finish Workflow' --workflow "MergeQueueCI" --ci |& tee ./ci/tmp/job.log
           fi
-
-##########################################################################################
-##################################### ALTINITY JOBS ######################################
-##########################################################################################
-  GrypeScanServer:
-    needs: [config_workflow, docker_server_image]
-    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIHNlcnZlciBpbWFnZQ==') }}
-    strategy:
-      fail-fast: false
-      matrix:
-        suffix: ['', '-alpine']
-    uses: ./.github/workflows/grype_scan.yml
-    secrets: inherit
-    with:
-      docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-      tag-suffix: ${{ matrix.suffix }}
-  GrypeScanKeeper:
-      needs: [config_workflow, docker_keeper_image]
-      if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
-      uses: ./.github/workflows/grype_scan.yml
-      secrets: inherit
-      with:
-        docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-
-  RegressionTestsRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester
-      commit: aa4204a74b901a0f2ea7c9f1d631d98221554fb1
-      arch: release
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-  RegressionTestsAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression') && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_aarch64')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: aa4204a74b901a0f2ea7c9f1d631d98221554fb1
-      arch: aarch64
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-
-  SignRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign release
-      runner_type: altinity-style-checker
-      data: ${{ needs.config_workflow.outputs.data }}
-  SignAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign aarch64
-      runner_type: altinity-style-checker-aarch64
-      data: ${{ needs.config_workflow.outputs.data }}
-
-  FinishCIReport:
-    if: ${{ !cancelled() }}
-    needs:
-      - config_workflow
-      - dockers_build_amd
-      - dockers_build_arm
-      - style_check
-      - fast_test
-      - build_amd_binary
-      - finish_workflow
-      - SignRelease
-      - SignAarch64
-      - RegressionTestsRelease
-      - RegressionTestsAarch64
-      - GrypeScanServer
-      - GrypeScanKeeper
-    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
-    steps:
-      - name: Check out repository code
-        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
-        with:
-          clear-repository: true
-      - name: Finalize workflow report
-        if: ${{ !cancelled() }}
-        uses: ./.github/actions/create_workflow_report
-        with:
-          workflow_config: ${{ needs.config_workflow.outputs.data }}
-          final: true

.github/workflows/nightly_fuzzers.yml

@@ -246,99 +246,3 @@ jobs:
           else
             python3 -m praktika run 'libFuzzer tests' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
           fi
-
-##########################################################################################
-##################################### ALTINITY JOBS ######################################
-##########################################################################################
-  GrypeScanServer:
-    needs: [config_workflow, docker_server_image]
-    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIHNlcnZlciBpbWFnZQ==') }}
-    strategy:
-      fail-fast: false
-      matrix:
-        suffix: ['', '-alpine']
-    uses: ./.github/workflows/grype_scan.yml
-    secrets: inherit
-    with:
-      docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-      tag-suffix: ${{ matrix.suffix }}
-  GrypeScanKeeper:
-      needs: [config_workflow, docker_keeper_image]
-      if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
-      uses: ./.github/workflows/grype_scan.yml
-      secrets: inherit
-      with:
-        docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-
-  RegressionTestsRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester
-      commit: aa4204a74b901a0f2ea7c9f1d631d98221554fb1
-      arch: release
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-  RegressionTestsAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression') && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_aarch64')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: aa4204a74b901a0f2ea7c9f1d631d98221554fb1
-      arch: aarch64
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-
-  SignRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign release
-      runner_type: altinity-style-checker
-      data: ${{ needs.config_workflow.outputs.data }}
-  SignAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign aarch64
-      runner_type: altinity-style-checker-aarch64
-      data: ${{ needs.config_workflow.outputs.data }}
-
-  FinishCIReport:
-    if: ${{ !cancelled() }}
-    needs:
-      - config_workflow
-      - dockers_build_amd
-      - dockers_build_arm
-      - build_fuzzers
-      - libfuzzer_tests
-      - SignRelease
-      - SignAarch64
-      - RegressionTestsRelease
-      - RegressionTestsAarch64
-      - GrypeScanServer
-      - GrypeScanKeeper
-    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
-    steps:
-      - name: Check out repository code
-        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
-        with:
-          clear-repository: true
-      - name: Finalize workflow report
-        if: ${{ !cancelled() }}
-        uses: ./.github/actions/create_workflow_report
-        with:
-          workflow_config: ${{ needs.config_workflow.outputs.data }}
-          final: true