fix: keep column statistics of all NULL column (#16753)

* fix: keep column statistics of all NULL column

* add logic test

* fix typos reported by typos-cli

Author: dantengsky

src/common/arrow/src/arrow/compute/merge_sort/mod.rs

@@ -491,7 +491,7 @@ pub fn build_comparator<'a>(
 }
 
 /// returns a comparison function between any two arrays of each pair of arrays, according to `SortOptions`.
-/// Implementing custom `build_compare_fn` for unsupportd data types.
+/// Implementing custom `build_compare_fn` for unsupported data types.
 pub fn build_comparator_impl<'a>(
     pairs: &'a [(&'a [&'a dyn Array], &SortOptions)],
     build_compare_fn: &dyn Fn(&dyn Array, &dyn Array) -> Result<DynComparator>,

src/common/arrow/src/arrow/compute/sort/lex_sort.rs

@@ -175,7 +175,7 @@ pub fn lexsort_to_indices<I: Index>(
 
 /// Sorts a list of [`SortColumn`] into a non-nullable [`PrimitiveArray`]
 /// representing the indices that would sort the columns.
-/// Implementing custom `build_compare_fn` for unsupportd data types.
+/// Implementing custom `build_compare_fn` for unsupported data types.
 pub fn lexsort_to_indices_impl<I: Index>(
     columns: &[SortColumn],
     limit: Option<usize>,

src/common/io/src/cursor_ext/cursor_read_bytes_ext.rs

@@ -112,8 +112,8 @@ where T: AsRef<[u8]>
         if available.is_empty() {
             return 0;
         }
-        for (index, byt) in available.iter().enumerate() {
-            if !f(*byt) {
+        for (index, bytes) in available.iter().enumerate() {
+            if !f(*bytes) {
                 self.consume(index);
                 return index;
             }

src/query/script/src/ir.rs

@@ -84,7 +84,7 @@ pub enum ScriptIR {
         condition: VarRef,
         to_label: LabelRef,
     },
-    /// Uncoditionally jumps to a specified label.
+    /// Unconditionally jumps to a specified label.
     Goto { to_label: LabelRef },
     /// Returns from the script.
     Return,

src/query/storages/common/table_meta/src/meta/v2/statistics.rs

@@ -393,14 +393,24 @@ impl<'de> serde::de::Visitor<'de> for ColStatsVisitor {
 
         while let Some(key) = access.next_key::<ColumnId>()? {
             if let Ok(value) = access.next_value::<ColumnStatistics>() {
-                let data_type = value.max.as_ref().infer_data_type();
-                if supported_stat_type(&data_type) {
+                if value.max.is_null() && value.min.is_null() {
+                    // If scalar values of min and max are all NULL, they should be retained.
+                    //
+                    // This ensures that columns with only NULL values have their column statistics
+                    // recorded, which is essential for pruning on these columns, and without this,
+                    // column statistics like NDV (Number of Distinct Values) and null_count
+                    // would be missing as well.
                     map.insert(key, value);
                 } else {
-                    info!(
-                        "column of id {} is excluded from column statistics, unsupported data type {}",
-                        key, data_type
-                    );
+                    let data_type = value.max.as_ref().infer_data_type();
+                    if supported_stat_type(&data_type) {
+                        map.insert(key, value);
+                    } else {
+                        info!(
+                            "column of id {} is excluded from column statistics, unsupported data type {}",
+                            key, data_type
+                        );
+                    }
                 }
             }
         }

tests/sqllogictests/suites/no_table_meta_cache/col_stats_of_all_null.test

@@ -0,0 +1,53 @@
+statement ok
+create or replace database col_stats_all_null;
+
+statement ok
+use col_stats_all_null;
+
+
+statement ok
+create or replace table t(c int) STORAGE_FORMAT=parquet;
+
+statement ok
+insert into t values(NULL);
+
+# segments should be pruned
+query T
+explain select * from t where c > 6;
+----
+Filter
+├── output columns: [t.c (#0)]
+├── filters: [is_true(t.c (#0) > 6)]
+├── estimated rows: 0.00
+└── TableScan
+    ├── table: default.col_stats_all_null.t
+    ├── output columns: [c (#0)]
+    ├── read rows: 0
+    ├── read size: 0
+    ├── partitions total: 1
+    ├── partitions scanned: 0
+    ├── pruning stats: [segments: <range pruning: 1 to 0>]
+    ├── push downs: [filters: [is_true(t.c (#0) > 6)], limit: NONE]
+    └── estimated rows: 1.00
+
+
+statement ok
+create or replace table t(c int) STORAGE_FORMAT=native;
+
+statement ok
+insert into t values(NULL);
+
+# segments should be pruned
+query T
+explain select * from t where c > 6;
+----
+TableScan
+├── table: default.col_stats_all_null.t
+├── output columns: [c (#0)]
+├── read rows: 0
+├── read size: 0
+├── partitions total: 1
+├── partitions scanned: 0
+├── pruning stats: [segments: <range pruning: 1 to 0>]
+├── push downs: [filters: [is_true(t.c (#0) > 6)], limit: NONE]
+└── estimated rows: 0.00