fix: collect stats fail with string starts with large unicodes. (#18523)

Author: youngsofun

src/query/service/tests/it/storages/fuse/statistics.rs

@@ -43,7 +43,7 @@ use databend_common_sql::ApproxDistinctColumns;
 use databend_common_storages_fuse::io::build_column_hlls;
 use databend_common_storages_fuse::statistics::reducers::reduce_block_metas;
 use databend_common_storages_fuse::statistics::Trim;
-use databend_common_storages_fuse::statistics::STATS_REPLACEMENT_CHAR;
+use databend_common_storages_fuse::statistics::END_OF_UNICODE_RANGE;
 use databend_common_storages_fuse::statistics::STATS_STRING_PREFIX_LEN;
 use databend_common_storages_fuse::FuseStorageFormat;
 use databend_query::storages::fuse::io::TableMetaLocationGenerator;
@@ -593,7 +593,7 @@ fn is_degenerated_case(value: &str) -> bool {
     let prefixed_with_irreplaceable_chars = value
         .chars()
         .take(STATS_STRING_PREFIX_LEN)
-        .all(|c| c >= STATS_REPLACEMENT_CHAR);
+        .all(|c| c >= END_OF_UNICODE_RANGE);
 
     larger_than_prefix_len && prefixed_with_irreplaceable_chars
 }

src/query/storages/fuse/src/io/write/stream/column_statistics_builder.rs

@@ -340,7 +340,9 @@ where
             if let Some(v) = T::upcast_scalar_with_type(v, &self.data_type).trim_max() {
                 v
             } else {
-                return Err(ErrorCode::Internal("Unable to trim string"));
+                return Err(ErrorCode::Internal(
+                    "Unable to trim string: first 16 chars are all replacement_point".to_string(),
+                ));
             }
         } else {
             Scalar::Null

src/query/storages/fuse/src/statistics/column_statistic.rs

@@ -168,7 +168,7 @@ pub trait Trim: Sized {
     fn may_be_trimmed(&self) -> bool;
 }
 
-pub const STATS_REPLACEMENT_CHAR: char = '\u{FFFD}';
+pub const END_OF_UNICODE_RANGE: char = '\u{10FFFF}';
 pub const STATS_STRING_PREFIX_LEN: usize = 16;
 
 impl Trim for Scalar {
@@ -220,7 +220,7 @@ impl Trim for Scalar {
                     // in reversed order, break at the first one we met
                     let mut idx = None;
                     for (i, c) in sliced.char_indices().rev() {
-                        if c < STATS_REPLACEMENT_CHAR {
+                        if c < END_OF_UNICODE_RANGE {
                             idx = Some(i);
                             break;
                         }
@@ -235,7 +235,7 @@ impl Trim for Scalar {
                         if i < replacement_point {
                             r.push(c)
                         } else {
-                            r.push(STATS_REPLACEMENT_CHAR);
+                            r.push(END_OF_UNICODE_RANGE);
                         }
                     }
 
@@ -253,3 +253,52 @@ impl Trim for Scalar {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use databend_common_expression::Scalar;
+
+    use crate::statistics::Trim;
+    use crate::statistics::END_OF_UNICODE_RANGE;
+    use crate::statistics::STATS_STRING_PREFIX_LEN;
+
+    #[test]
+    fn test_trim_max() {
+        {
+            let invalid = END_OF_UNICODE_RANGE
+                .to_string()
+                .repeat(STATS_STRING_PREFIX_LEN + 1);
+            assert_eq!(Scalar::String(invalid).trim_max(), None);
+        }
+
+        {
+            let s = END_OF_UNICODE_RANGE
+                .to_string()
+                .repeat(STATS_STRING_PREFIX_LEN);
+            let scalar = Scalar::String(s.clone());
+            assert_eq!(scalar.clone().trim_max(), Some(scalar));
+        }
+
+        {
+            let s = '👍'.to_string().repeat(STATS_STRING_PREFIX_LEN + 1);
+            let res = Scalar::String(s.clone()).trim_max();
+            let mut exp = '👍'.to_string().repeat(STATS_STRING_PREFIX_LEN - 1);
+            exp.push(END_OF_UNICODE_RANGE);
+            assert_eq!(res, Some(Scalar::String(exp)));
+        }
+
+        {
+            let mut s = '👍'.to_string().repeat(STATS_STRING_PREFIX_LEN - 1);
+            s.push(END_OF_UNICODE_RANGE);
+            s.push(END_OF_UNICODE_RANGE);
+
+            let res = Scalar::String(s.clone()).trim_max();
+
+            let mut exp = '👍'.to_string().repeat(STATS_STRING_PREFIX_LEN - 2);
+            exp.push(END_OF_UNICODE_RANGE);
+            exp.push(END_OF_UNICODE_RANGE);
+
+            assert_eq!(res, Some(Scalar::String(exp)));
+        }
+    }
+}

src/query/storages/fuse/src/statistics/mod.rs

@@ -27,7 +27,7 @@ pub use column_statistic::calc_column_distinct_of_values;
 pub use column_statistic::gen_columns_statistics;
 pub use column_statistic::scalar_min_max;
 pub use column_statistic::Trim;
-pub use column_statistic::STATS_REPLACEMENT_CHAR;
+pub use column_statistic::END_OF_UNICODE_RANGE;
 pub use column_statistic::STATS_STRING_PREFIX_LEN;
 pub use reducers::merge_statistics;
 pub use reducers::reduce_block_metas;