Merge branch 'master' into test_attr

Author: nickitat

.github/actions/common_setup/action.yml

@@ -28,29 +28,3 @@ runs:
       run: |
           # to remove every leftovers
           sudo rm -fr "$TEMP_PATH" && mkdir -p "$TEMP_PATH"
-    # https://github.com/google/sanitizers/issues/856
-    - name: Tune vm.mmap_rnd_bits for sanitizers
-      shell: bash
-      run: |
-          sudo sysctl vm.mmap_rnd_bits
-          sudo sysctl vm.mmap_rnd_bits=28
-    - name: Tune kernel.task_delayacct=1 for OSIOWaitMicroseconds
-      shell: bash
-      run: |
-          sudo sysctl kernel.task_delayacct
-          sudo sysctl kernel.task_delayacct=1
-    - name: Avoid relying on internal AWS DNS
-      shell: bash
-      run: |
-          fqdn=$(hostname --fqdn)
-          echo "Current FQDN: $fqdn"
-          echo "127.0.0.1 $fqdn" | sudo tee -a /etc/hosts
-          resolvectl flush-caches
-          resolvectl query "$fqdn"
-          # Sometimes it tries to resolve the address using NSes:
-          #
-          #     ip-172-31-33-98.ec2.internal has address 127.0.0.1
-          #     Host ip-172-31-33-98.ec2.internal not found: 3(NXDOMAIN)
-          #
-          # So let's ignore the error
-          host "$fqdn" || true

.github/workflows/jepsen.yml

@@ -6,6 +6,9 @@ on:
     - cron: 13 4 * * *
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}
+
 env:
   PYTHONUNBUFFERED: 1
   CHECKOUT_REF: ${{ github.head_ref }}
@@ -128,7 +131,7 @@ jobs:
 
   build_amd_binary:
     runs-on: [self-hosted, builder]
-    needs: [config_workflow]
+    needs: [config_workflow, dockers_build_amd]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9iaW5hcnkp') }}
     name: "Build (amd_binary)"
     outputs:

.github/workflows/nightly_jepsen.yml

@@ -11,7 +11,7 @@ env:
   PYTHONUNBUFFERED: 1
   DISABLE_CI_MERGE_COMMIT: ${{ vars.DISABLE_CI_MERGE_COMMIT || '0' }}
   DISABLE_CI_CACHE: ${{ vars.DISABLE_CI_CACHE || '0' }}
-  CHECKOUT_REF: ${{ vars.DISABLE_CI_MERGE_COMMIT == '1' && '' || github.event.pull_request.head.sha }}
+  CHECKOUT_REF: ${{ vars.DISABLE_CI_MERGE_COMMIT == '1' && github.event.pull_request.head.sha || '' }}
 
 # Allow updating GH commit statuses and PR comments to post an actual job reports link
 permissions: write-all
@@ -3817,82 +3817,6 @@ jobs:
             python3 -m praktika run 'Upgrade check (debug)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
           fi
 
-  clickbench_release:
-    runs-on: [self-hosted, func-tester]
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'Q2xpY2tCZW5jaCAocmVsZWFzZSk=') }}
-    name: "ClickBench (release)"
-    outputs:
-      data: ${{ steps.run.outputs.DATA }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ env.CHECKOUT_REF }}
-
-      - name: Prepare env script
-        run: |
-          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
-          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
-          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
-          export PYTHONPATH=./ci:.:
-          cat > ./ci/tmp/workflow_config_pr.json << 'EOF'
-          ${{ needs.config_workflow.outputs.data }}
-          EOF
-          cat > ./ci/tmp/workflow_status.json << 'EOF'
-          ${{ toJson(needs) }}
-          EOF
-          ENV_SETUP_SCRIPT_EOF
-
-      - name: Run
-        id: run
-        run: |
-          . ./ci/tmp/praktika_setup_env.sh
-          set -o pipefail
-          if command -v ts &> /dev/null; then
-            python3 -m praktika run 'ClickBench (release)' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
-          else
-            python3 -m praktika run 'ClickBench (release)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
-          fi
-
-  clickbench_aarch64:
-    runs-on: [self-hosted, func-tester-aarch64]
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'Q2xpY2tCZW5jaCAoYWFyY2g2NCk=') }}
-    name: "ClickBench (aarch64)"
-    outputs:
-      data: ${{ steps.run.outputs.DATA }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ env.CHECKOUT_REF }}
-
-      - name: Prepare env script
-        run: |
-          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
-          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
-          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
-          export PYTHONPATH=./ci:.:
-          cat > ./ci/tmp/workflow_config_pr.json << 'EOF'
-          ${{ needs.config_workflow.outputs.data }}
-          EOF
-          cat > ./ci/tmp/workflow_status.json << 'EOF'
-          ${{ toJson(needs) }}
-          EOF
-          ENV_SETUP_SCRIPT_EOF
-
-      - name: Run
-        id: run
-        run: |
-          . ./ci/tmp/praktika_setup_env.sh
-          set -o pipefail
-          if command -v ts &> /dev/null; then
-            python3 -m praktika run 'ClickBench (aarch64)' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
-          else
-            python3 -m praktika run 'ClickBench (aarch64)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
-          fi
-
   ast_fuzzer_debug:
     runs-on: [self-hosted, func-tester]
     needs: [config_workflow, build_amd_debug]
@@ -4387,9 +4311,123 @@ jobs:
             python3 -m praktika run 'Performance Comparison (release, 3/3)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
           fi
 
+  performance_comparison_aarch64_1_3:
+    runs-on: [self-hosted, func-tester-aarch64]
+    needs: [config_workflow, build_arm_release]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'UGVyZm9ybWFuY2UgQ29tcGFyaXNvbiAoYWFyY2g2NCwgMS8zKQ==') }}
+    name: "Performance Comparison (aarch64, 1/3)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_pr.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Performance Comparison (aarch64, 1/3)' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Performance Comparison (aarch64, 1/3)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
+          fi
+
+  performance_comparison_aarch64_2_3:
+    runs-on: [self-hosted, func-tester-aarch64]
+    needs: [config_workflow, build_arm_release]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'UGVyZm9ybWFuY2UgQ29tcGFyaXNvbiAoYWFyY2g2NCwgMi8zKQ==') }}
+    name: "Performance Comparison (aarch64, 2/3)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_pr.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Performance Comparison (aarch64, 2/3)' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Performance Comparison (aarch64, 2/3)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
+          fi
+
+  performance_comparison_aarch64_3_3:
+    runs-on: [self-hosted, func-tester-aarch64]
+    needs: [config_workflow, build_arm_release]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'UGVyZm9ybWFuY2UgQ29tcGFyaXNvbiAoYWFyY2g2NCwgMy8zKQ==') }}
+    name: "Performance Comparison (aarch64, 3/3)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_pr.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Performance Comparison (aarch64, 3/3)' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Performance Comparison (aarch64, 3/3)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
+          fi
+
   finish_workflow:
     runs-on: [self-hosted, style-checker-aarch64]
-    needs: [config_workflow, dockers_build_arm, dockers_build_amd, style_check, docs_check, fast_test, build_amd_debug, build_amd_release, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_release, build_arm_asan, build_amd_coverage, build_arm_binary, build_amd_tidy, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, unit_tests_binary, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, docker_server_image, docker_keeper_image, install_packages_release, install_packages_aarch64, compatibility_check_release, compatibility_check_aarch64, stateless_tests_asan_1_2, stateless_tests_asan_2_2, stateless_tests_release, stateless_tests_release_old_analyzer_s3_databasereplicated_1_2, stateless_tests_release_old_analyzer_s3_databasereplicated_2_2, stateless_tests_release_parallelreplicas_s3_storage, stateless_tests_debug, stateless_tests_tsan_1_3, stateless_tests_tsan_2_3, stateless_tests_tsan_3_3, stateless_tests_msan_1_4, stateless_tests_msan_2_4, stateless_tests_msan_3_4, stateless_tests_msan_4_4, stateless_tests_ubsan, stateless_tests_coverage_1_6, stateless_tests_coverage_2_6, stateless_tests_coverage_3_6, stateless_tests_coverage_4_6, stateless_tests_coverage_5_6, stateless_tests_coverage_6_6, stateless_tests_debug_s3_storage, stateless_tests_tsan_s3_storage_1_3, stateless_tests_tsan_s3_storage_2_3, stateless_tests_tsan_s3_storage_3_3, stateless_tests_aarch64, stateless_tests_azure_asan_1_3, stateless_tests_azure_asan_2_3, stateless_tests_azure_asan_3_3, bugfix_validation, stateless_tests_asan_flaky_check, integration_tests_asan_old_analyzer_1_6, integration_tests_asan_old_analyzer_2_6, integration_tests_asan_old_analyzer_3_6, integration_tests_asan_old_analyzer_4_6, integration_tests_asan_old_analyzer_5_6, integration_tests_asan_old_analyzer_6_6, integration_tests_release_1_4, integration_tests_release_2_4, integration_tests_release_3_4, integration_tests_release_4_4, integration_tests_aarch64_1_4, integration_tests_aarch64_2_4, integration_tests_aarch64_3_4, integration_tests_aarch64_4_4, integration_tests_tsan_1_6, integration_tests_tsan_2_6, integration_tests_tsan_3_6, integration_tests_tsan_4_6, integration_tests_tsan_5_6, integration_tests_tsan_6_6, integration_tests_asan_flaky_check, stress_test_debug, stress_test_tsan, stress_test_asan, stress_test_ubsan, stress_test_msan, upgrade_check_asan, upgrade_check_tsan, upgrade_check_msan, upgrade_check_debug, clickbench_release, clickbench_aarch64, ast_fuzzer_debug, ast_fuzzer_asan, ast_fuzzer_tsan, ast_fuzzer_msan, ast_fuzzer_ubsan, buzzhouse_debug, buzzhouse_asan, buzzhouse_tsan, buzzhouse_msan, buzzhouse_ubsan, performance_comparison_release_1_3, performance_comparison_release_2_3, performance_comparison_release_3_3]
+    needs: [config_workflow, dockers_build_arm, dockers_build_amd, style_check, docs_check, fast_test, build_amd_debug, build_amd_release, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_release, build_arm_asan, build_amd_coverage, build_arm_binary, build_amd_tidy, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, unit_tests_binary, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, docker_server_image, docker_keeper_image, install_packages_release, install_packages_aarch64, compatibility_check_release, compatibility_check_aarch64, stateless_tests_asan_1_2, stateless_tests_asan_2_2, stateless_tests_release, stateless_tests_release_old_analyzer_s3_databasereplicated_1_2, stateless_tests_release_old_analyzer_s3_databasereplicated_2_2, stateless_tests_release_parallelreplicas_s3_storage, stateless_tests_debug, stateless_tests_tsan_1_3, stateless_tests_tsan_2_3, stateless_tests_tsan_3_3, stateless_tests_msan_1_4, stateless_tests_msan_2_4, stateless_tests_msan_3_4, stateless_tests_msan_4_4, stateless_tests_ubsan, stateless_tests_coverage_1_6, stateless_tests_coverage_2_6, stateless_tests_coverage_3_6, stateless_tests_coverage_4_6, stateless_tests_coverage_5_6, stateless_tests_coverage_6_6, stateless_tests_debug_s3_storage, stateless_tests_tsan_s3_storage_1_3, stateless_tests_tsan_s3_storage_2_3, stateless_tests_tsan_s3_storage_3_3, stateless_tests_aarch64, stateless_tests_azure_asan_1_3, stateless_tests_azure_asan_2_3, stateless_tests_azure_asan_3_3, bugfix_validation, stateless_tests_asan_flaky_check, integration_tests_asan_old_analyzer_1_6, integration_tests_asan_old_analyzer_2_6, integration_tests_asan_old_analyzer_3_6, integration_tests_asan_old_analyzer_4_6, integration_tests_asan_old_analyzer_5_6, integration_tests_asan_old_analyzer_6_6, integration_tests_release_1_4, integration_tests_release_2_4, integration_tests_release_3_4, integration_tests_release_4_4, integration_tests_aarch64_1_4, integration_tests_aarch64_2_4, integration_tests_aarch64_3_4, integration_tests_aarch64_4_4, integration_tests_tsan_1_6, integration_tests_tsan_2_6, integration_tests_tsan_3_6, integration_tests_tsan_4_6, integration_tests_tsan_5_6, integration_tests_tsan_6_6, integration_tests_asan_flaky_check, stress_test_debug, stress_test_tsan, stress_test_asan, stress_test_ubsan, stress_test_msan, upgrade_check_asan, upgrade_check_tsan, upgrade_check_msan, upgrade_check_debug, ast_fuzzer_debug, ast_fuzzer_asan, ast_fuzzer_tsan, ast_fuzzer_msan, ast_fuzzer_ubsan, buzzhouse_debug, buzzhouse_asan, buzzhouse_tsan, buzzhouse_msan, buzzhouse_ubsan, performance_comparison_release_1_3, performance_comparison_release_2_3, performance_comparison_release_3_3, performance_comparison_aarch64_1_3, performance_comparison_aarch64_2_3, performance_comparison_aarch64_3_3]
     if: ${{ !cancelled() }}
     name: "Finish Workflow"
     outputs:

.github/workflows/pull_request.yml

@@ -0,0 +1,45 @@
+name: WeeklyOfficialDocker
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+"on":
+  ### Disabled until the workflow will work ###
+  # schedule:
+  #   # Fridays 09:55 UTC
+  #   - cron: '55 9 * * 5'
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        description: 'Dry run'
+        required: false
+        default: true
+        type: boolean
+
+concurrency:
+  group: official-docker-library
+
+jobs:
+  UpdateOfficialDocker:
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          fetch-depth: 0 # to get all tags and their content
+      - name: Debug Info
+        uses: ./.github/actions/debug
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          DRY_RUN=${{ github.event.inputs.dry-run && '--dry-run' || '' }}
+          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
+          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
+          RCSK
+          EOF
+      - name: Update official docker repos
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 official_docker_update.py -vv $DRY_RUN