Merge branch 'antalya-25.8' into fp_antalya_25_8_list_objects_cache

Author: arthurpassos

.github/workflows/master.yml

@@ -4173,7 +4173,7 @@ jobs:
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_binary]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression')}}
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).custom_data.ci_exclude_tags, 'regression')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:
@@ -4185,7 +4185,7 @@ jobs:
       workflow_config: ${{ needs.config_workflow.outputs.data }}
   RegressionTestsAarch64:
     needs: [config_workflow, build_arm_binary]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression') && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_aarch64')}}
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).custom_data.ci_exclude_tags, 'regression') && !contains(fromJson(needs.config_workflow.outputs.data).custom_data.ci_exclude_tags, 'aarch64')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:

.github/workflows/pull_request.yml

@@ -4129,7 +4129,7 @@ jobs:
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_binary]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression')}}
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).custom_data.ci_exclude_tags, 'regression')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:
@@ -4141,7 +4141,7 @@ jobs:
       workflow_config: ${{ needs.config_workflow.outputs.data }}
   RegressionTestsAarch64:
     needs: [config_workflow, build_arm_binary]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression') && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_aarch64')}}
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).custom_data.ci_exclude_tags, 'regression') && !contains(fromJson(needs.config_workflow.outputs.data).custom_data.ci_exclude_tags, 'aarch64')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:

.github/workflows/regression.yml

@@ -122,6 +122,7 @@ env:
   pr_number: ${{ github.event.number }}
   event_name: ${{ github.event_name }}
   version: ${{ fromJson(inputs.workflow_config).custom_data.version.string }}
+  SKIP_LIST: ${{ join(fromJson(inputs.workflow_config).custom_data.ci_exclude_tags, '|') || '' }}
 
 jobs:
   runner_labels_setup:
@@ -147,6 +148,9 @@ jobs:
           input: ${{ inputs.runner_type }}
 
   Common:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'common')
     strategy:
       fail-fast: false
       matrix:
@@ -211,6 +215,9 @@ jobs:
           path: ${{ env.artifact_paths}}
 
   AggregateFunctions:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'aggregate_functions')
     strategy:
       fail-fast: false
       matrix:
@@ -276,6 +283,9 @@ jobs:
           name: ${{ env.SUITE }}-${{ matrix.PART }}-${{ inputs.arch }}-artifacts
           path: ${{ env.artifact_paths}}
   Alter:
+   if: |
+     fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+     contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'alter')
    strategy:
      fail-fast: false
      matrix:
@@ -348,6 +358,9 @@ jobs:
           path: ${{ env.artifact_paths}}
 
   Benchmark:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'benchmark')
     strategy:
       fail-fast: false
       matrix:
@@ -421,6 +434,9 @@ jobs:
           path: ${{ env.artifact_paths }}
 
   ClickHouseKeeper:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'clickhouse_keeper')
     strategy:
       fail-fast: false
       matrix:
@@ -490,6 +506,9 @@ jobs:
           path: ${{ env.artifact_paths }}
 
   Iceberg:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'iceberg')
     strategy:
       fail-fast: false
       matrix:
@@ -561,6 +580,9 @@ jobs:
           name: ${{ env.SUITE }}-${{ matrix.PART }}-${{ inputs.arch }}-artifacts
           path: ${{ env.artifact_paths}}
   LDAP:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'ldap')
     strategy:
       fail-fast: false
       matrix:
@@ -625,6 +647,9 @@ jobs:
           path: ${{ env.artifact_paths }}
 
   Parquet:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'parquet')
     needs: [runner_labels_setup]
     runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }}
     timeout-minutes: ${{ inputs.timeout_minutes }}
@@ -685,6 +710,9 @@ jobs:
           path: ${{ env.artifact_paths }}
 
   ParquetS3:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'parquet')
     strategy:
       fail-fast: false
       matrix:
@@ -756,6 +784,9 @@ jobs:
 
 
   RBAC:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'rbac')
     strategy:
       fail-fast: false
       matrix:
@@ -821,6 +852,9 @@ jobs:
           name: ${{ env.SUITE }}-${{ matrix.PART }}-${{ inputs.arch }}-artifacts
           path: ${{ env.artifact_paths}}
   SSLServer:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'ssl_server')
     strategy:
       fail-fast: false
       matrix:
@@ -887,6 +921,9 @@ jobs:
           path: ${{ env.artifact_paths}}
 
   S3:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 's3')
     strategy:
       fail-fast: false
       matrix:
@@ -969,6 +1006,9 @@ jobs:
           path: ${{ env.artifact_paths}}
 
   TieredStorage:
+    if: |
+      fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null ||
+      contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'tiered_storage')
     strategy:
       fail-fast: false
       matrix:

ci/jobs/scripts/workflow_hooks/filter_job.py

@@ -1,6 +1,3 @@
-import re
-from functools import lru_cache
-
 from ci.defs.defs import JobNames
 from ci.defs.job_configs import JobConfigs
 from ci.jobs.scripts.workflow_hooks.new_tests_check import (
@@ -52,15 +49,6 @@ def only_docs(changed_files):
 _info_cache = None
 
 
-@lru_cache
-def get_ci_exclude_tags(pr_body):
-    pattern = r"(#|- \[x\] +<!---ci_exclude_)([|\w]+)"
-    matches = []
-    for match in re.findall(pattern, pr_body):
-        matches.extend(match[-1].split("|"))
-    return matches
-
-
 def should_skip_job(job_name):
     global _info_cache
     if _info_cache is None:
@@ -179,7 +167,7 @@ def should_skip_job(job_name):
             return False, ""
         return True, "Skipped, not labeled with 'pr-performance'"
 
-    ci_exclude_tags = get_ci_exclude_tags(_info_cache.pr_body)
+    ci_exclude_tags = _info_cache.get_kv_data("ci_exclude_tags") or []
     for tag in ci_exclude_tags:
         if tag in job_name:
             return True, f"Skipped, job name includes excluded tag '{tag}'"

ci/jobs/scripts/workflow_hooks/parse_ci_tags.py

@@ -0,0 +1,18 @@
+import re
+
+from ci.praktika.info import Info
+
+
+def get_ci_tags(pr_body, tag_prefix):
+    pattern = rf"(- \[x\] +<!---{tag_prefix}_)([|\w]+)"
+    matches = []
+    for match in re.findall(pattern, pr_body):
+        matches.extend(match[-1].split("|"))
+    return matches
+
+
+if __name__ == "__main__":
+    info = Info()
+
+    info.store_kv_data("ci_exclude_tags", get_ci_tags(info.pr_body, "ci_exclude"))
+    info.store_kv_data("ci_regression_jobs", get_ci_tags(info.pr_body, "ci_regression"))

ci/praktika/native_jobs.py

@@ -398,9 +398,23 @@ def check_affected_jobs():
 
             affected_artifacts = []
             unaffected_jobs_with_artifacts = {}
-            # NOTE (strtgbb): We always want the build artifacts for our report and regression tests.
+            all_required_artifacts = set()
+
+            # NOTE (strtgbb): We always want these build artifacts for our report and regression tests.
             # If we make FinishCIReport and regression tests into praktika jobs, we can remove this.
-            all_required_artifacts = set(["CH_AMD_RELEASE", "CH_ARM_RELEASE"])
+            if "CIReport" in workflow.additional_jobs:
+                all_required_artifacts.update(["CH_AMD_RELEASE", "CH_ARM_RELEASE"])
+            if (
+                "Regression" in workflow.additional_jobs
+                and "regression"
+                not in workflow_config.custom_data.get("ci_exclude_tags", [])
+            ):
+                all_required_artifacts.update(["CH_AMD_BINARY"])
+                if "aarch64" not in workflow_config.custom_data.get(
+                    "ci_exclude_tags", []
+                ):
+                    all_required_artifacts.update(["CH_ARM_BINARY"])
+            print(f"Including artifacts for custom jobs [{all_required_artifacts}]")
 
             for job in workflow.jobs:
                 # Skip native Praktika jobs

ci/praktika/yaml_additional_templates.py

@@ -63,7 +63,7 @@ class AltinityWorkflowTemplates:
         "Regression": r"""
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_binary]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression')}}
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).custom_data.ci_exclude_tags, 'regression')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:
@@ -75,7 +75,7 @@ class AltinityWorkflowTemplates:
       workflow_config: ${{ needs.config_workflow.outputs.data }}
   RegressionTestsAarch64:
     needs: [config_workflow, build_arm_binary]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression') && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_aarch64')}}
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).custom_data.ci_exclude_tags, 'regression') && !contains(fromJson(needs.config_workflow.outputs.data).custom_data.ci_exclude_tags, 'aarch64')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:

ci/workflows/master.py

@@ -63,6 +63,7 @@
     pre_hooks=[
         "python3 ./ci/jobs/scripts/workflow_hooks/store_data.py",
         "python3 ./ci/jobs/scripts/workflow_hooks/version_log.py",
+        "python3 ./ci/jobs/scripts/workflow_hooks/parse_ci_tags.py",
         # "python3 ./ci/jobs/scripts/workflow_hooks/merge_sync_pr.py", # NOTE (strtgbb): we don't do this
     ],
     workflow_filter_hooks=[should_skip_job],

ci/workflows/pull_request.py

@@ -133,6 +133,7 @@
         "python3 ./ci/jobs/scripts/workflow_hooks/version_log.py",
         # "python3 ./ci/jobs/scripts/workflow_hooks/quick_sync.py", # NOTE (strtgbb): we don't do this
         # "python3 ./ci/jobs/scripts/workflow_hooks/team_notifications.py",
+        "python3 ./ci/jobs/scripts/workflow_hooks/parse_ci_tags.py",
     ],
     workflow_filter_hooks=[should_skip_job],
     post_hooks=[

programs/server/Server.cpp

@@ -158,6 +158,10 @@
 #   include <azure/core/diagnostics/logger.hpp>
 #endif
 
+#if USE_PARQUET
+#   include <Processors/Formats/Impl/ParquetFileMetaDataCache.h>
+#endif
+
 
 #include <incbin.h>
 /// A minimal file used when the server is run without installation
@@ -342,6 +346,7 @@ namespace ServerSetting
     extern const ServerSettingsUInt64 object_storage_list_objects_cache_ttl;
     extern const ServerSettingsUInt64 object_storage_list_objects_cache_size;
     extern const ServerSettingsUInt64 object_storage_list_objects_cache_max_entries;
+    extern const ServerSettingsUInt64 input_format_parquet_metadata_cache_max_size;
 }
 
 namespace ErrorCodes
@@ -2535,6 +2540,10 @@ try
 
     auto replicas_reconnector = ReplicasReconnector::init(global_context);
 
+#if USE_PARQUET
+    ParquetFileMetaDataCache::instance()->setMaxSizeInBytes(server_settings[ServerSetting::input_format_parquet_metadata_cache_max_size]);
+#endif
+
     /// Set current database name before loading tables and databases because
     /// system logs may copy global context.
     std::string default_database = server_settings[ServerSetting::default_database];