Backport ClickHouse#87136 to 25.8: Fix maps and arrays field ids reading parquet

Author: robot-clickhouse

src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp

@@ -1121,7 +1121,7 @@ static ColumnWithTypeAndName readNonNullableColumnFromArrowColumn(
 
             auto nested_column = readColumnFromArrowColumn(arrow_nested_column,
                 column_name,
-                Nested::concatenateName(full_column_name, "list.element"),
+                full_column_name,
                 dictionary_infos,
                 nested_type_hint,
                 is_nested_nullable_column,
@@ -1211,12 +1211,9 @@ static ColumnWithTypeAndName readNonNullableColumnFromArrowColumn(
                 {
                     chassert(clickhouse_columns_to_parquet);
 
-                    auto column_to_search = full_column_name;
-                    if (column_to_search.ends_with("list.element"))
-                        column_to_search = column_to_search.substr(0, column_to_search.size() - 13);
                     /// Full name of the parquet column.
                     /// For example, if the column name is "a" and the field name in the structure is "b", the full name will be "a.b".
-                    auto full_name = clickhouse_columns_to_parquet->at(column_to_search);
+                    auto full_name = clickhouse_columns_to_parquet->at(full_column_name);
                     full_name += "." + field_name;
                     if (auto it = parquet_columns_to_clickhouse->find(full_name); it != parquet_columns_to_clickhouse->end())
                     {

src/Processors/Formats/Impl/ArrowFieldIndexUtil.h

@@ -179,7 +179,7 @@ class ArrowFieldIndexUtil
             const auto * map_type = static_cast<arrow::MapType *>(field_type.get());
             auto index_snapshot = current_start_index;
             current_start_index += countIndicesForType(map_type->key_type());
-            calculateFieldIndices(*map_type->item_field(), field_name, current_start_index, result, name_prefix);
+            calculateFieldIndices(*map_type->item_field(), Nested::concatenateName(field_name, "value"), current_start_index, result, name_prefix);
             index_info.first = index_snapshot;
         }
         else
@@ -217,7 +217,9 @@ class ArrowFieldIndexUtil
                     if (clickhouse_to_parquet_names)
                     {
                         if (auto it = clickhouse_to_parquet_names->find(full_name); it != clickhouse_to_parquet_names->end())
+                        {
                             full_name = it->second;
+                        }
                     }
 
                     findRequiredIndices(Nested::concatenateName(name, field_name), full_name, header_index, field_type, field_indices, added_indices, required_indices, file, clickhouse_to_parquet_names);
@@ -233,7 +235,7 @@ class ArrowFieldIndexUtil
         else if (const auto * type_map = typeid_cast<const DB::DataTypeMap *>(nested_type.get()))
         {
             findRequiredIndices(name, transformed_name, header_index, type_map->getKeyType(), field_indices, added_indices, required_indices, file, clickhouse_to_parquet_names);
-            findRequiredIndices(name, transformed_name, header_index, type_map->getValueType(), field_indices, added_indices, required_indices, file, clickhouse_to_parquet_names);
+            findRequiredIndices(Nested::concatenateName(name, "value"), Nested::concatenateName(transformed_name, "value"), header_index, type_map->getValueType(), field_indices, added_indices, required_indices, file, clickhouse_to_parquet_names);
             return;
         }
         auto it = field_indices.find(transformed_name);

src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp

@@ -69,6 +69,15 @@ namespace ErrorCodes
 
 namespace
 {
+String removeListElement(const String & value)
+{
+    const String pattern = ".list.element";
+    String result = value;
+    size_t pos;
+    while ((pos = result.find(pattern)) != std::string::npos)
+        result.erase(pos, pattern.size());
+    return result;
+}
 
 
 void traverseAllFields(const parquet::schema::NodePtr & node, std::unordered_map<Int64, String> & fields_mapping, const String & current_path = "")
@@ -80,7 +89,7 @@ void traverseAllFields(const parquet::schema::NodePtr & node, std::unordered_map
             traverseAllFields(group->field(i), fields_mapping, Nested::concatenateName(current_path, group->name()));
     }
     int field_id = node->field_id();
-    fields_mapping[field_id] = Nested::concatenateName(current_path, node->name());
+    fields_mapping[field_id] = removeListElement(Nested::concatenateName(current_path, node->name()));
 }
 
 }

src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp

@@ -72,7 +72,7 @@ void traverseComplexType(Poco::JSON::Object::Ptr type, std::unordered_map<String
     {
         auto element_id = type->getValue<Int64>(Iceberg::f_element_id);
         if (type->isObject(Iceberg::f_element))
-            traverseComplexType(type->getObject(Iceberg::f_element), result, Nested::concatenateName(current_path, "list.element"));
+            traverseComplexType(type->getObject(Iceberg::f_element), result, current_path);
         result[current_path] = element_id;
         return;
     }

tests/queries/0_stateless/03581_iceberg_struct_fields_ids.reference

@@ -1 +1,2 @@
 2025-06-04	('100022','2025-06-04 18:40:56.000000','2025-06-09 21:19:00.364000')	100022
+[1,2,3]	{1:'Pavel',2:'Ivanov'}	[(1,'pudge'),(2,'1000-7')]

tests/queries/0_stateless/03581_iceberg_struct_fields_ids.sql

@@ -4,3 +4,4 @@
 -- This table was created by spark, but with changed column names in parquet
 -- The purpose of this test is to verify that a column with a structured type with different names in Parquet and Iceberg metadata will be read correctly.
 SELECT * FROM icebergS3(s3_conn, filename='field_ids_struct_test', SETTINGS iceberg_metadata_table_uuid = '149ecc15-7afc-4311-86b3-3a4c8d4ec08e') ORDER BY ALL;
+SELECT * FROM icebergS3(s3_conn, filename='field_ids_complex_test', SETTINGS iceberg_metadata_table_uuid = 'd4b695ca-ceeb-4537-8a2a-eee90dc6e313') ORDER BY ALL;

tests/queries/0_stateless/data_minio/field_ids_complex_test/data/00000-1-a4207d9e-e52b-45e4-9f8f-40d60dda5485-0-00001.parquet

@@ -0,0 +1,109 @@
+{
+  "format-version" : 2,
+  "table-uuid" : "d4b695ca-ceeb-4537-8a2a-eee90dc6e313",
+  "location" : "s3a://test/field_ids_struct_test/metadata/field_ids_complex_test",
+  "last-sequence-number" : 1,
+  "last-updated-ms" : 1757661733693,
+  "last-column-id" : 9,
+  "current-schema-id" : 0,
+  "schemas" : [ {
+    "type" : "struct",
+    "schema-id" : 0,
+    "fields" : [ {
+      "id" : 1,
+      "name" : "col-1",
+      "required" : false,
+      "type" : {
+        "type" : "list",
+        "element-id" : 4,
+        "element" : "long",
+        "element-required" : false
+      }
+    }, {
+      "id" : 2,
+      "name" : "col-2",
+      "required" : false,
+      "type" : {
+        "type" : "map",
+        "key-id" : 5,
+        "key" : "long",
+        "value-id" : 6,
+        "value" : "string",
+        "value-required" : false
+      }
+    }, {
+      "id" : 3,
+      "name" : "col-3",
+      "required" : false,
+      "type" : {
+        "type" : "list",
+        "element-id" : 7,
+        "element" : {
+          "type" : "struct",
+          "fields" : [ {
+            "id" : 8,
+            "name" : "id",
+            "required" : false,
+            "type" : "long"
+          }, {
+            "id" : 9,
+            "name" : "name",
+            "required" : false,
+            "type" : "string"
+          } ]
+        },
+        "element-required" : false
+      }
+    } ]
+  } ],
+  "default-spec-id" : 0,
+  "partition-specs" : [ {
+    "spec-id" : 0,
+    "fields" : [ ]
+  } ],
+  "last-partition-id" : 999,
+  "default-sort-order-id" : 0,
+  "sort-orders" : [ {
+    "order-id" : 0,
+    "fields" : [ ]
+  } ],
+  "properties" : {
+    "owner" : "scanhex12",
+    "write.parquet.compression-codec" : "zstd"
+  },
+  "current-snapshot-id" : 607752583403487091,
+  "refs" : {
+    "main" : {
+      "snapshot-id" : 607752583403487091,
+      "type" : "branch"
+    }
+  },
+  "snapshots" : [ {
+    "sequence-number" : 1,
+    "snapshot-id" : 607752583403487091,
+    "timestamp-ms" : 1757661733693,
+    "summary" : {
+      "operation" : "append",
+      "spark.app.id" : "local-1757661643417",
+      "added-data-files" : "1",
+      "added-records" : "1",
+      "added-files-size" : "1957",
+      "changed-partition-count" : "1",
+      "total-records" : "1",
+      "total-files-size" : "1957",
+      "total-data-files" : "1",
+      "total-delete-files" : "0",
+      "total-position-deletes" : "0",
+      "total-equality-deletes" : "0"
+    },
+    "manifest-list" : "s3a://test/field_ids_struct_test/metadata/field_ids_complex_test/metadata/snap-607752583403487091-1-140c8dff-1d83-4841-bc40-9aa85205b555.avro",
+    "schema-id" : 0
+  } ],
+  "statistics" : [ ],
+  "partition-statistics" : [ ],
+  "snapshot-log" : [ {
+    "timestamp-ms" : 1757661733693,
+    "snapshot-id" : 607752583403487091
+  } ],
+  "metadata-log" : [ ]
+}