update

Author: zhyass

Cargo.lock

@@ -491,7 +491,7 @@ sha1 = "0.10.5"
 sha2 = "0.10.8"
 simdutf8 = "0.1.4"
 similar = "2.7.0"
-simple_hll = { version = "0.0.1", features = ["serde_borsh"] }
+simple_hll = { version = "0.0.4", features = ["serde_borsh"] }
 simsearch = "0.2"
 siphasher = "0.3"
 sled = { version = "0.34", default-features = false }

Cargo.toml

@@ -37,6 +37,7 @@ prometheus-client = { workspace = true }
 regex = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true }
+simple_hll = { workspace = true, features = ["serde_borsh"] }
 thiserror = { workspace = true }
 url = { workspace = true }
 

src/common/storage/Cargo.toml

@@ -85,6 +85,7 @@ pub use histogram::HistogramBucket;
 pub use histogram::DEFAULT_HISTOGRAM_BUCKETS;
 pub use merge::MutationStatus;
 pub use meta_hll::MetaHLL;
+pub use meta_hll::MetaHLL12;
 pub use multi_table_insert::MultiTableInsertStatus;
 pub use statistics::Datum;
 pub use statistics::F64;

src/common/storage/src/lib.rs

@@ -18,6 +18,8 @@ use std::hash::Hash;
 
 use ahash::RandomState;
 
+pub type MetaHLL12 = simple_hll::HyperLogLog<12>;
+
 const P: usize = 7_usize;
 const Q: usize = 64 - P;
 const M: usize = 1 << P;
@@ -51,6 +53,11 @@ impl MetaHLL {
         }
     }
 
+    pub fn with_registers(registers: Vec<u8>) -> Self {
+        assert_eq!(registers.len(), M);
+        Self { registers }
+    }
+
     /// Adds an hash to the MetaHLL.
     /// hash value is dertermined by caller
     #[inline]
@@ -67,7 +74,7 @@ impl MetaHLL {
 
     /// Adds an object to the MetaHLL.
     /// Though we could pass different types into this method, caller should notice that
-    pub fn add_object<T: Hash>(&mut self, obj: &T) {
+    pub fn add_object<T: ?Sized + Hash>(&mut self, obj: &T) {
         let hash = SEED.hash_one(obj);
         self.add_hash(hash);
     }
@@ -110,6 +117,18 @@ impl MetaHLL {
     pub fn num_empty_registers(&self) -> usize {
         self.registers.iter().filter(|x| **x == 0).count()
     }
+
+    pub fn from_hll(hll: MetaHLL12) -> Self {
+        let registers = hll.get_registers();
+        let mut new_registers = vec![0; M];
+        let group_size = registers.len() / M;
+        for i in 0..M {
+            for j in 0..group_size {
+                new_registers[i] = new_registers[i].max(registers[i * group_size + j]);
+            }
+        }
+        Self::with_registers(new_registers)
+    }
 }
 
 /// Helper function sigma as defined in
@@ -366,4 +385,20 @@ mod tests {
         }
         compare_with_delta(hll.count(), 1000);
     }
+
+    #[test]
+    fn test_from_hll() {
+        let mut hll = MetaHLL12::new();
+        for i in 0..100_000 {
+            hll.add_object(&i);
+        }
+
+        let hll = MetaHLL::from_hll(hll);
+        let count = hll.count();
+        let error_rate = 1.04 / ((M as f64).sqrt());
+        let diff = count as f64 / 100_000f64;
+
+        assert!(diff >= 1.0 - error_rate);
+        assert!(diff <= 1.0 + error_rate);
+    }
 }

src/common/storage/src/meta_hll.rs

@@ -23,6 +23,7 @@ use databend_common_expression::types::number::NumberScalar;
 use databend_common_expression::ColumnId;
 use databend_common_expression::Scalar;
 use databend_common_io::prelude::borsh_deserialize_from_slice;
+use databend_common_storage::MetaHLL12;
 use databend_common_storages_fuse::io::MetaReaders;
 use databend_common_storages_fuse::io::MetaWriter;
 use databend_common_storages_fuse::statistics::reducers::merge_statistics_mut;
@@ -33,7 +34,6 @@ use databend_query::sql::plans::Plan;
 use databend_query::sql::Planner;
 use databend_query::test_kits::*;
 use databend_storages_common_cache::LoadParams;
-use databend_storages_common_table_meta::meta::MetaHLL12;
 use databend_storages_common_table_meta::meta::SegmentInfo;
 use databend_storages_common_table_meta::meta::Statistics;
 use databend_storages_common_table_meta::meta::TableSnapshot;

src/query/service/tests/it/storages/fuse/operations/table_analyze.rs

@@ -30,7 +30,6 @@ parquet = { workspace = true }
 rmp-serde = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-simple_hll = { workspace = true, features = ["serde_borsh"] }
 snap = { workspace = true, optional = true }
 typetag = { workspace = true }
 zstd = { workspace = true }

src/query/storages/common/table_meta/Cargo.toml

@@ -21,7 +21,6 @@ pub use v2::ColumnStatistics;
 pub use v2::DraftVirtualBlockMeta;
 pub use v2::DraftVirtualColumnMeta;
 pub use v2::ExtendedBlockMeta;
-pub use v2::MetaHLL12;
 pub use v2::SegmentStatistics;
 pub use v2::Statistics;
 pub use v2::VirtualBlockMeta;

src/query/storages/common/table_meta/src/meta/current/mod.rs

@@ -32,5 +32,4 @@ pub use statistics::AdditionalStatsMeta;
 pub use statistics::ClusterStatistics;
 pub use statistics::ColumnStatistics;
 pub use statistics::Statistics;
-pub use table_snapshot_statistics::MetaHLL12;
 pub use table_snapshot_statistics::TableSnapshotStatistics;

src/query/storages/common/table_meta/src/meta/v2/mod.rs

@@ -15,6 +15,7 @@
 use std::collections::HashMap;
 
 use databend_common_expression::ColumnId;
+use databend_common_storage::MetaHLL12;
 use serde::Deserialize;
 use serde::Serialize;
 
@@ -23,8 +24,6 @@ use crate::meta::FormatVersion;
 use crate::meta::SnapshotId;
 use crate::meta::Versioned;
 
-pub type MetaHLL12 = simple_hll::HyperLogLog<12>;
-
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct TableSnapshotStatistics {
     /// format version of snapshot