feat: enable block stream write (#18285)

* improve stream write

* fix virtual column builder

* add block statistics

* remove null in bloom index builder

* use metahll

* avoid large string

* fix

* use segment level stats

* add test

* remove unused code

* fix review comment

* fix test

---------

Co-authored-by: Bohu <[email protected]>

Author: zhyass

Cargo.lock

@@ -10,9 +10,11 @@ edition = { workspace = true }
 storage-hdfs = ["databend-common-meta-app/storage-hdfs"]
 
 [dependencies]
+ahash = { workspace = true }
 anyhow = { workspace = true }
 arrow-schema = { workspace = true }
 async-backtrace = { workspace = true }
+borsh = { workspace = true }
 chrono = { workspace = true }
 dashmap = { workspace = true, features = ["serde"] }
 databend-common-ast = { workspace = true }

src/common/storage/Cargo.toml

@@ -71,6 +71,7 @@ pub use stage::STDIN_FD;
 mod copy;
 mod histogram;
 mod merge;
+mod meta_hll;
 mod metrics_layer;
 mod multi_table_insert;
 mod statistics;
@@ -83,6 +84,7 @@ pub use histogram::Histogram;
 pub use histogram::HistogramBucket;
 pub use histogram::DEFAULT_HISTOGRAM_BUCKETS;
 pub use merge::MutationStatus;
+pub use meta_hll::MetaHLL;
 pub use multi_table_insert::MultiTableInsertStatus;
 pub use statistics::Datum;
 pub use statistics::F64;