Merge branch 'antalya-25.8' into export_replicated_mt_partition_v2

Author: arthurpassos

.github/workflows/master.yml

@@ -4178,7 +4178,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester
-      commit: 3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568
+      commit: fc19ce3a7322a10ab791de755c950a56744a12e7
       arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300
@@ -4190,7 +4190,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: 3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568
+      commit: fc19ce3a7322a10ab791de755c950a56744a12e7
       arch: aarch64
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300

.github/workflows/pull_request.yml

@@ -4134,7 +4134,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester
-      commit: 3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568
+      commit: fc19ce3a7322a10ab791de755c950a56744a12e7
       arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300
@@ -4146,7 +4146,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: 3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568
+      commit: fc19ce3a7322a10ab791de755c950a56744a12e7
       arch: aarch64
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300

.gitmodules

@@ -6,7 +6,7 @@
 	url = https://github.com/Thalhammer/jwt-cpp
 [submodule "contrib/zstd"]
 	path = contrib/zstd
-	url = https://github.com/facebook/zstd
+	url = https://github.com/ClickHouse/zstd.git
 [submodule "contrib/lz4"]
 	path = contrib/lz4
 	url = https://github.com/lz4/lz4
@@ -45,7 +45,7 @@
 	url = https://github.com/ClickHouse/arrow
 [submodule "contrib/thrift"]
 	path = contrib/thrift
-	url = https://github.com/apache/thrift
+	url = https://github.com/ClickHouse/thrift.git
 [submodule "contrib/libhdfs3"]
 	path = contrib/libhdfs3
 	url = https://github.com/ClickHouse/libhdfs3

ci/praktika/native_jobs.py

@@ -18,6 +18,7 @@
 from .runtime import RunConfig
 from .settings import Settings
 from .utils import Shell, Utils
+from ci.defs.defs import ArtifactNames
 
 assert Settings.CI_CONFIG_RUNS_ON
 
@@ -403,17 +404,22 @@ def check_affected_jobs():
             # NOTE (strtgbb): We always want these build artifacts for our report and regression tests.
             # If we make FinishCIReport and regression tests into praktika jobs, we can remove this.
             if "CIReport" in workflow.additional_jobs:
-                all_required_artifacts.update(["CH_AMD_RELEASE", "CH_ARM_RELEASE"])
+                all_required_artifacts.update(
+                    [
+                        ArtifactNames.CH_AMD_RELEASE,
+                        ArtifactNames.CH_ARM_RELEASE,
+                    ]
+                )
             if (
                 "Regression" in workflow.additional_jobs
                 and "regression"
                 not in workflow_config.custom_data.get("ci_exclude_tags", [])
             ):
-                all_required_artifacts.update(["CH_AMD_BINARY"])
+                all_required_artifacts.update([ArtifactNames.CH_AMD_BINARY])
                 if "aarch64" not in workflow_config.custom_data.get(
                     "ci_exclude_tags", []
                 ):
-                    all_required_artifacts.update(["CH_ARM_BINARY"])
+                    all_required_artifacts.update([ArtifactNames.CH_ARM_BINARY])
             print(f"Including artifacts for custom jobs [{all_required_artifacts}]")
 
             for job in workflow.jobs:

ci/praktika/yaml_additional_templates.py

@@ -35,7 +35,7 @@ class AltinityWorkflowTemplates:
           echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
 """
     # Additional jobs
-    REGRESSION_HASH = "3fbe58a0ebe8fa5f97b7f36c45a2a69b1d3b6568"
+    REGRESSION_HASH = "fc19ce3a7322a10ab791de755c950a56744a12e7"
     ALTINITY_JOBS = {
         "GrypeScan": r"""
   GrypeScanServer:

src/IO/S3/Client.cpp

@@ -703,19 +703,35 @@ Client::doRequest(RequestType & request, RequestFn request_fn) const
             continue;
         }
 
-        if (error.GetResponseCode() != Aws::Http::HttpResponseCode::MOVED_PERMANENTLY)
+        /// IllegalLocationConstraintException may indicate that we are working with an opt-in region (e.g. me-south-1)
+        /// In that case, we need to update the region and try again
+        bool is_illegal_constraint_exception = error.GetExceptionName() == "IllegalLocationConstraintException";
+        if (error.GetResponseCode() != Aws::Http::HttpResponseCode::MOVED_PERMANENTLY && !is_illegal_constraint_exception)
             return result;
 
         // maybe we detect a correct region
-        if (!detect_region)
+        bool new_region_detected = false;
+        if (!detect_region || is_illegal_constraint_exception)
         {
             if (auto region = GetErrorMarshaller()->ExtractRegion(error); !region.empty() && region != explicit_region)
             {
+                LOG_INFO(log, "Detected new region: {}", region);
                 request.overrideRegion(region);
                 insertRegionOverride(bucket, region);
+                new_region_detected = true;
             }
         }
 
+        /// special handling for opt-in regions
+        if (new_region_detected && is_illegal_constraint_exception && initial_endpoint.substr(11) == "amazonaws.com")
+        {
+            S3::URI new_uri(initial_endpoint);
+            new_uri.addRegionToURI(request.getRegionOverride());
+            found_new_endpoint = true;
+            request.overrideURI(new_uri);
+            continue;
+        }
+
         // we possibly got new location, need to try with that one
         auto new_uri = getURIFromError(error);
         if (!new_uri)

src/Storages/System/StorageSystemTables.cpp

@@ -607,18 +607,25 @@ class TablesBlockSource : public ISource
                 if (columns_mask[src_index++])
                 {
                     bool inserted = false;
-                    // Extract from specific DataLake metadata if suitable
-                    if (auto * obj = dynamic_cast<StorageObjectStorageCluster *>(table.get()))
+                    try
                     {
-                        if (auto * dl_meta = obj->getExternalMetadata(context))
+                        // Extract from specific DataLake metadata if suitable
+                        if (auto * obj = dynamic_cast<StorageObjectStorageCluster *>(table.get()))
                         {
-                            if (auto p = dl_meta->partitionKey(context); p.has_value())
+                            if (auto * dl_meta = obj->getExternalMetadata(context))
                             {
-                                res_columns[res_index++]->insert(*p);
-                                inserted = true;
+                                if (auto p = dl_meta->partitionKey(context); p.has_value())
+                                {
+                                    res_columns[res_index++]->insert(*p);
+                                    inserted = true;
+                                }
                             }
                         }
-
+                    }
+                    catch (const Exception &)
+                    {
+                        /// Failed to get info from Iceberg. It's not critical, just log it.
+                        tryLogCurrentException("StorageSystemTables");
                     }
 
                     if (!inserted)
@@ -634,18 +641,26 @@ class TablesBlockSource : public ISource
                 {
                     bool inserted = false;
 
-                    // Extract from specific DataLake metadata if suitable
-                    if (auto * obj = dynamic_cast<StorageObjectStorageCluster *>(table.get()))
+                    try
                     {
-                        if (auto * dl_meta = obj->getExternalMetadata(context))
+                        // Extract from specific DataLake metadata if suitable
+                        if (auto * obj = dynamic_cast<StorageObjectStorageCluster *>(table.get()))
                         {
-                            if (auto p = dl_meta->sortingKey(context); p.has_value())
+                            if (auto * dl_meta = obj->getExternalMetadata(context))
                             {
-                                res_columns[res_index++]->insert(*p);
-                                inserted = true;
+                                if (auto p = dl_meta->sortingKey(context); p.has_value())
+                                {
+                                    res_columns[res_index++]->insert(*p);
+                                    inserted = true;
+                                }
                             }
                         }
                     }
+                    catch (const Exception &)
+                    {
+                        /// Failed to get info from Iceberg. It's not critical, just log it.
+                        tryLogCurrentException("StorageSystemTables");
+                    }
 
                     if (!inserted)
                     {