Merge pull request #1037 from Altinity/backports/25.6.5/87515_and_87621_do_not_put_hive_partition_columns_in_format_header

Antalya 25.6.5 - Backport of ClickHouse#87515 and ClickHouse#87621 - Never not put hive columns in format_header / Do not read hive columns as LC by default

Author: Enmk

src/Storages/HivePartitioningUtils.cpp

@@ -9,6 +9,7 @@
 #include <Formats/EscapingRuleUtils.h>
 #include <Formats/FormatFactory.h>
 #include <Processors/Chunk.h>
+#include <DataTypes/IDataType.h>
 
 namespace DB
 {
@@ -85,7 +86,7 @@ NamesAndTypesList extractHivePartitionColumnsFromPath(
         {
             if (const auto type = tryInferDataTypeByEscapingRule(value, format_settings ? *format_settings : getFormatSettings(context), FormatSettings::EscapingRule::Raw))
             {
-                if (type->canBeInsideLowCardinality())
+                if (type->canBeInsideLowCardinality() && isStringOrFixedString(type))
                 {
                     hive_partition_columns_to_read_from_file_path.emplace_back(key, std::make_shared<DataTypeLowCardinality>(type));
                 }

src/Storages/prepareReadingFromFormat.cpp

@@ -86,7 +86,13 @@ ReadFromFormatInfo prepareReadingFromFormat(
     }
 
     /// Create header for InputFormat with columns that will be read from the data.
-    info.format_header = storage_snapshot->getSampleBlockForColumns(info.columns_description.getNamesOfPhysical());
+    for (const auto & column : info.columns_description)
+    {
+        /// Never read hive partition columns from the data file. This fixes https://github.com/ClickHouse/ClickHouse/issues/87515
+        if (!hive_parameters.hive_partition_columns_to_read_from_file_path_map.contains(column.name))
+            info.format_header.insert(ColumnWithTypeAndName{column.type, column.name});
+    }
+
     info.serialization_hints = getSerializationHintsForFileLikeStorage(storage_snapshot->metadata, context);
     return info;
 }

tests/queries/0_stateless/03203_hive_style_partitioning.reference

@@ -31,7 +31,7 @@ Elizabeth	Delgado
 Elizabeth	Cross
 42	2020-01-01
 [1,2,3]	42.42
-Array(Int64)	LowCardinality(Float64)
+Array(Int64)	Float64
 101
 2071
 2071

tests/queries/0_stateless/03631_hive_columns_not_in_format_header.reference

@@ -0,0 +1,2 @@
+1
+raw_blob	String					

tests/queries/0_stateless/03631_hive_columns_not_in_format_header.sql

@@ -0,0 +1,13 @@
+-- Tags: no-parallel, no-fasttest, no-random-settings
+
+INSERT INTO FUNCTION s3(
+    s3_conn,
+    filename='03631',
+    format=Parquet,
+    partition_strategy='hive',
+    partition_columns_in_data_file=1) PARTITION BY (year, country) SELECT 'Brazil' as country, 2025 as year, 1 as id;
+
+-- distinct because minio isn't cleaned up
+SELECT count(distinct year) FROM s3(s3_conn, filename='03631/**.parquet', format=RawBLOB) SETTINGS use_hive_partitioning=1;
+
+DESCRIBE s3(s3_conn, filename='03631/**.parquet', format=RawBLOB) SETTINGS use_hive_partitioning=1;