feat: Use max/min to estimate ndv (#17468)

* feat: Use max/min to estimate ndv

Signed-off-by: Xuanwo <[email protected]>

* Address cases where max doesn't vaild

Signed-off-by: Xuanwo <[email protected]>

* Fix show databases

Signed-off-by: Xuanwo <[email protected]>

* Fix ndv

Signed-off-by: Xuanwo <[email protected]>

* Fix overflow

Signed-off-by: Xuanwo <[email protected]>

* Fix scan

Signed-off-by: Xuanwo <[email protected]>

* Fix explain

Signed-off-by: Xuanwo <[email protected]>

* Fix test

Signed-off-by: Xuanwo <[email protected]>

* test show tables

Signed-off-by: Xuanwo <[email protected]>

---------

Signed-off-by: Xuanwo <[email protected]>

Author: Xuanwo

src/query/sql/src/planner/binder/ddl/database.rs

@@ -59,7 +59,7 @@ impl Binder {
             limit,
         } = stmt;
 
-        let mut select_builder = SelectBuilder::from("system.databases");
+        let mut select_builder = SelectBuilder::from("default.system.databases");
 
         let ctl = if let Some(ctl) = catalog {
             normalize_identifier(ctl, &self.name_resolution_ctx).name
@@ -101,7 +101,7 @@ impl Binder {
         stmt: &ShowDropDatabasesStmt,
     ) -> Result<Plan> {
         let ShowDropDatabasesStmt { catalog, limit } = stmt;
-        let mut select_builder = SelectBuilder::from("system.databases_with_history");
+        let mut select_builder = SelectBuilder::from("default.system.databases_with_history");
 
         let ctl = if let Some(ctl) = catalog {
             normalize_identifier(ctl, &self.name_resolution_ctx).name

src/query/sql/src/planner/plans/scan.rs

@@ -24,6 +24,7 @@ use databend_common_catalog::table_context::TableContext;
 use databend_common_exception::ErrorCode;
 use databend_common_exception::Result;
 use databend_common_expression::TableSchemaRef;
+use databend_common_storage::Datum;
 use databend_common_storage::Histogram;
 use databend_common_storage::DEFAULT_HISTOGRAM_BUCKETS;
 use databend_storages_common_table_meta::table::ChangeType;
@@ -240,9 +241,23 @@ impl Operator for Scan {
                 // ndv could be `None`, we will use `num_rows - null_count` as ndv instead.
                 //
                 // NOTE: don't touch the original num_rows, since it will be used in other places.
-                let ndv = col_stat
+                let mut ndv = col_stat
                     .ndv
                     .unwrap_or_else(|| num_rows.saturating_sub(col_stat.null_count));
+
+                // Alter ndv based on min and max if the datum is uint or int.
+                match (&max, &min) {
+                    (Datum::UInt(m), Datum::UInt(n)) if m >= n => ndv = ndv.min(m - n + 1),
+                    (Datum::Int(m), Datum::Int(n)) if m >= n => {
+                        ndv = ndv.min(m.saturating_add(1).saturating_sub(*n) as u64)
+                    }
+                    _ => {
+                        if max == min {
+                            ndv = 1
+                        }
+                    }
+                };
+
                 let histogram = if let Some(histogram) = self.statistics.histograms.get(k)
                     && histogram.is_some()
                 {

tests/sqllogictests/suites/tpch_iceberg/prune.test

@@ -82,11 +82,11 @@ explain select 1 from  ctl.tpch.lineitem where l_orderkey > 1 and l_commitdate =
 EvalScalar
 ├── output columns: [1 (#16)]
 ├── expressions: [1]
-├── estimated rows: 1.00
+├── estimated rows: 243.54
 └── Filter
     ├── output columns: []
     ├── filters: [is_true(lineitem.l_orderkey (#0) > 1), is_true(lineitem.l_commitdate (#11) = '1992-01-22')]
-    ├── estimated rows: 1.00
+    ├── estimated rows: 243.54
     └── TableScan
         ├── table: ctl.tpch.lineitem
         ├── output columns: [l_orderkey (#0), l_commitdate (#11)]

tests/sqllogictests/suites/tpch_iceberg/utils.test

@@ -0,0 +1,18 @@
+statement ok
+DROP CATALOG IF EXISTS ctl;
+
+statement ok
+CREATE CATALOG ctl
+TYPE=ICEBERG
+CONNECTION=(
+    TYPE='rest'
+    ADDRESS='http://127.0.0.1:8181'
+    WAREHOUSE='s3://iceberg-tpch'
+    "s3.region"='us-east-1'
+    "s3.endpoint"='http://127.0.0.1:9000'
+);
+
+query T
+show databases from ctl;
+----
+tpch