Merge branch 'antalya-25.8' into feature/antalya-25.8/iceberg_local_cluster

Author: ianton-ru

.github/workflows/master.yml

@@ -4178,7 +4178,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester
-      commit: 3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568
+      commit: fc19ce3a7322a10ab791de755c950a56744a12e7
       arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300
@@ -4190,7 +4190,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: 3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568
+      commit: fc19ce3a7322a10ab791de755c950a56744a12e7
       arch: aarch64
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300

.github/workflows/pull_request.yml

@@ -4134,7 +4134,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester
-      commit: 3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568
+      commit: fc19ce3a7322a10ab791de755c950a56744a12e7
       arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300
@@ -4146,7 +4146,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: 3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568
+      commit: fc19ce3a7322a10ab791de755c950a56744a12e7
       arch: aarch64
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300

.gitmodules

@@ -6,7 +6,7 @@
 	url = https://github.com/Thalhammer/jwt-cpp
 [submodule "contrib/zstd"]
 	path = contrib/zstd
-	url = https://github.com/facebook/zstd
+	url = https://github.com/ClickHouse/zstd.git
 [submodule "contrib/lz4"]
 	path = contrib/lz4
 	url = https://github.com/lz4/lz4
@@ -45,7 +45,7 @@
 	url = https://github.com/ClickHouse/arrow
 [submodule "contrib/thrift"]
 	path = contrib/thrift
-	url = https://github.com/apache/thrift
+	url = https://github.com/ClickHouse/thrift.git
 [submodule "contrib/libhdfs3"]
 	path = contrib/libhdfs3
 	url = https://github.com/ClickHouse/libhdfs3

ci/praktika/native_jobs.py

@@ -18,6 +18,7 @@
 from .runtime import RunConfig
 from .settings import Settings
 from .utils import Shell, Utils
+from ci.defs.defs import ArtifactNames
 
 assert Settings.CI_CONFIG_RUNS_ON
 
@@ -403,17 +404,22 @@ def check_affected_jobs():
             # NOTE (strtgbb): We always want these build artifacts for our report and regression tests.
             # If we make FinishCIReport and regression tests into praktika jobs, we can remove this.
             if "CIReport" in workflow.additional_jobs:
-                all_required_artifacts.update(["CH_AMD_RELEASE", "CH_ARM_RELEASE"])
+                all_required_artifacts.update(
+                    [
+                        ArtifactNames.CH_AMD_RELEASE,
+                        ArtifactNames.CH_ARM_RELEASE,
+                    ]
+                )
             if (
                 "Regression" in workflow.additional_jobs
                 and "regression"
                 not in workflow_config.custom_data.get("ci_exclude_tags", [])
             ):
-                all_required_artifacts.update(["CH_AMD_BINARY"])
+                all_required_artifacts.update([ArtifactNames.CH_AMD_BINARY])
                 if "aarch64" not in workflow_config.custom_data.get(
                     "ci_exclude_tags", []
                 ):
-                    all_required_artifacts.update(["CH_ARM_BINARY"])
+                    all_required_artifacts.update([ArtifactNames.CH_ARM_BINARY])
             print(f"Including artifacts for custom jobs [{all_required_artifacts}]")
 
             for job in workflow.jobs:

ci/praktika/yaml_additional_templates.py

@@ -35,7 +35,7 @@ class AltinityWorkflowTemplates:
           echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
 """
     # Additional jobs
-    REGRESSION_HASH = "3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568"
+    REGRESSION_HASH = "fc19ce3a7322a10ab791de755c950a56744a12e7"
     ALTINITY_JOBS = {
         "GrypeScan": r"""
   GrypeScanServer:

src/Storages/MergeTree/MergeTreeData.cpp

@@ -213,6 +213,7 @@ namespace Setting
     extern const SettingsUInt64 min_bytes_to_use_direct_io;
     extern const SettingsBool export_merge_tree_part_overwrite_file_if_exists;
     extern const SettingsBool output_format_parallel_formatting;
+    extern const SettingsBool output_format_parquet_parallel_encoding;
 }
 
 namespace MergeTreeSetting
@@ -6244,7 +6245,9 @@ void MergeTreeData::exportPartToTable(const PartitionCommand & command, ContextP
             dest_storage->getStorageID(),
             part,
             query_context->getSettingsRef()[Setting::export_merge_tree_part_overwrite_file_if_exists],
-            query_context->getSettingsRef()[Setting::output_format_parallel_formatting]);
+            query_context->getSettingsRef()[Setting::output_format_parallel_formatting],
+            query_context->getSettingsRef()[Setting::output_format_parquet_parallel_encoding],
+            query_context->getSettingsRef()[Setting::max_threads]);
 
         std::lock_guard lock(export_manifests_mutex);
 
@@ -6292,6 +6295,8 @@ void MergeTreeData::exportPartToTableImpl(
     {
         auto context_copy = Context::createCopy(local_context);
         context_copy->setSetting("output_format_parallel_formatting", manifest.parallel_formatting);
+        context_copy->setSetting("output_format_parquet_parallel_encoding", manifest.parquet_parallel_encoding);
+        context_copy->setSetting("max_threads", manifest.max_threads);
 
         sink = destination_storage->import(
             manifest.data_part->name + "_" + manifest.data_part->checksums.getTotalChecksumHex(),

src/Storages/MergeTree/MergeTreeExportManifest.h

@@ -13,17 +13,24 @@ struct MergeTreeExportManifest
         const StorageID & destination_storage_id_,
         const DataPartPtr & data_part_,
         bool overwrite_file_if_exists_,
-        bool parallel_formatting_)
+        bool parallel_formatting_,
+        bool parallel_formatting_parquet_,
+        std::size_t max_threads_)
         : destination_storage_id(destination_storage_id_),
           data_part(data_part_),
           overwrite_file_if_exists(overwrite_file_if_exists_),
           parallel_formatting(parallel_formatting_),
+          parquet_parallel_encoding(parallel_formatting_parquet_),
+          max_threads(max_threads_),
           create_time(time(nullptr)) {}
 
     StorageID destination_storage_id;
     DataPartPtr data_part;
     bool overwrite_file_if_exists;
     bool parallel_formatting;
+    /// parquet has a different setting for parallel formatting
+    bool parquet_parallel_encoding;
+    std::size_t max_threads;
 
     time_t create_time;
     mutable bool in_progress = false;

src/Storages/MergeTree/MergeTreeSequentialSource.cpp

@@ -169,7 +169,8 @@ MergeTreeSequentialSource::MergeTreeSequentialSource(
             addThrottler(read_settings.local_throttler, context->getMergesThrottler());
             break;
         case Export:
-            read_settings.local_throttler = context->getExportsThrottler();
+            addThrottler(read_settings.local_throttler, context->getExportsThrottler());
+            addThrottler(read_settings.remote_throttler, context->getExportsThrottler());
             break;
     }
 

src/Storages/ObjectStorage/ObjectStorageFilePathGenerator.h

@@ -55,7 +55,7 @@ namespace DB
 
             result += raw_path;
 
-            if (raw_path.back() != '/')
+            if (!raw_path.empty() && raw_path.back() != '/')
             {
                 result += "/";
             }

src/Storages/ObjectStorage/StorageObjectStorage.cpp

@@ -1,3 +1,4 @@
+#include <optional>
 #include <thread>
 #include <Core/ColumnWithTypeAndName.h>
 #include <Storages/ObjectStorage/StorageObjectStorage.h>
@@ -507,7 +508,7 @@ SinkToStoragePtr StorageObjectStorage::import(
         destination_file_path,
         object_storage,
         configuration,
-        format_settings,
+        std::nullopt, /// passing nullopt to force rebuild for format_settings based on query context
         std::make_shared<const Block>(getInMemoryMetadataPtr()->getSampleBlock()),
         local_context);
 }