feat(cost-model): implement cost model derive statistics

optd-cost-model/Cargo.toml

@@ -18,6 +18,7 @@ itertools = "0.13"
 assert_approx_eq = "1.1.0"
 trait-variant = "0.1.2"
 tokio = { version = "1.0.1", features = ["macros", "rt-multi-thread"] }
+async-trait = "0.1"
 
 [dev-dependencies]
 crossbeam = "0.8"

optd-cost-model/src/cost/agg.rs

@@ -8,7 +8,7 @@ use crate::{
     cost_model::CostModelImpl,
     stats::DEFAULT_NUM_DISTINCT,
     storage::CostModelStorageManager,
-    CostModelError, CostModelResult, EstimatedStatistic, SemanticError,
+    CostModelResult, EstimatedStatistic, SemanticError,
 };
 
 impl<S: CostModelStorageManager> CostModelImpl<S> {
@@ -67,21 +67,16 @@ impl<S: CostModelStorageManager> CostModelImpl<S> {
 
 #[cfg(test)]
 mod tests {
-    use std::{collections::HashMap, ops::Deref};
+    use std::collections::HashMap;
 
     use crate::{
-        common::{
-            predicates::constant_pred::ConstantType,
-            properties::Attribute,
-            types::{GroupId, TableId},
-            values::Value,
-        },
-        cost_model::tests::{
-            attr_index, cnst, create_mock_cost_model, create_mock_cost_model_with_attr_types,
-            empty_list, empty_per_attr_stats, list, TestPerAttributeStats, TEST_ATTR1_BASE_INDEX,
-            TEST_ATTR2_BASE_INDEX, TEST_ATTR3_BASE_INDEX, TEST_GROUP1_ID, TEST_TABLE1_ID,
-        },
+        common::predicates::constant_pred::ConstantType,
         stats::{utilities::simple_map::SimpleMap, MostCommonValues, DEFAULT_NUM_DISTINCT},
+        test_utils::tests::{
+            attr_index, create_mock_cost_model_with_attr_types, empty_list, list,
+            TestPerAttributeStats, TEST_ATTR1_BASE_INDEX, TEST_ATTR2_BASE_INDEX,
+            TEST_ATTR3_BASE_INDEX, TEST_GROUP1_ID, TEST_TABLE1_ID,
+        },
         EstimatedStatistic,
     };
 

optd-cost-model/src/cost/filter/comp_op.rs

@@ -14,7 +14,7 @@ use crate::{
     cost_model::CostModelImpl,
     stats::{DEFAULT_EQ_SEL, DEFAULT_INEQ_SEL, UNIMPLEMENTED_SEL},
     storage::CostModelStorageManager,
-    CostModelResult, SemanticError,
+    CostModelResult,
 };
 
 impl<S: CostModelStorageManager> CostModelImpl<S> {

optd-cost-model/src/cost/filter/core.rs

@@ -98,16 +98,11 @@ mod tests {
                 bin_op_pred::BinOpType, constant_pred::ConstantType, log_op_pred::LogOpType,
                 un_op_pred::UnOpType,
             },
-            properties::Attribute,
             types::TableId,
             values::Value,
         },
-        cost_model::tests::*,
-        memo_ext::tests::MemoGroupInfo,
-        stats::{
-            utilities::{counter::Counter, simple_map::SimpleMap},
-            Distribution, MostCommonValues, DEFAULT_EQ_SEL,
-        },
+        stats::{utilities::simple_map::SimpleMap, Distribution, MostCommonValues, DEFAULT_EQ_SEL},
+        test_utils::tests::*,
     };
     use arrow_schema::DataType;
 
@@ -834,7 +829,7 @@ mod tests {
             0,
             0.0,
         );
-        let table_id = TableId(0);
+
         let cost_model = create_mock_cost_model_with_attr_types(
             vec![TEST_TABLE1_ID],
             vec![HashMap::from([(

optd-cost-model/src/cost/filter/in_list.rs

@@ -83,16 +83,9 @@ mod tests {
     use std::collections::HashMap;
 
     use crate::{
-        common::{
-            types::{GroupId, TableId},
-            values::Value,
-        },
-        cost_model::tests::*,
-        memo_ext::tests::MemoGroupInfo,
-        stats::{
-            utilities::{counter::Counter, simple_map::SimpleMap},
-            MostCommonValues,
-        },
+        common::values::Value,
+        stats::{utilities::simple_map::SimpleMap, MostCommonValues},
+        test_utils::tests::*,
     };
 
     #[tokio::test]

optd-cost-model/src/cost/filter/like.rs

@@ -113,15 +113,12 @@ mod tests {
     use std::collections::HashMap;
 
     use crate::{
-        common::{
-            types::{GroupId, TableId},
-            values::Value,
-        },
-        cost_model::tests::*,
+        common::values::Value,
         stats::{
             utilities::{counter::Counter, simple_map::SimpleMap},
             MostCommonValues, FIXED_CHAR_SEL_FACTOR, FULL_WILDCARD_SEL_FACTOR,
         },
+        test_utils::tests::*,
     };
 
     #[tokio::test]

optd-cost-model/src/cost/filter/log_op.rs

@@ -25,7 +25,7 @@ impl<S: CostModelStorageManager> CostModelImpl<S> {
                 let mut or_sel_neg = 1.0;
                 for child in children {
                     let selectivity = self.get_filter_selectivity(group_id, child.clone()).await?;
-                    or_sel_neg *= (1.0 - selectivity);
+                    or_sel_neg *= 1.0 - selectivity;
                 }
                 Ok(1.0 - or_sel_neg)
             }

optd-cost-model/src/cost/join/core.rs

@@ -7,13 +7,11 @@ use crate::{
         nodes::{ArcPredicateNode, JoinType, PredicateType, ReprPredicateNode},
         predicates::{
             attr_index_pred::AttrIndexPred,
-            bin_op_pred::BinOpType,
             list_pred::ListPred,
             log_op_pred::{LogOpPred, LogOpType},
         },
         properties::attr_ref::{
-            self, AttrRef, AttrRefs, BaseTableAttrRef, EqPredicate, GroupAttrRefs,
-            SemanticCorrelation,
+            AttrRef, AttrRefs, BaseTableAttrRef, EqPredicate, SemanticCorrelation,
         },
         types::GroupId,
     },
@@ -409,23 +407,24 @@ impl<S: CostModelStorageManager> CostModelImpl<S> {
 mod tests {
     use std::collections::HashMap;
 
+    use attr_ref::GroupAttrRefs;
+
     use crate::{
         common::{
-            predicates::{attr_index_pred, constant_pred::ConstantType},
-            properties::Attribute,
-            types::TableId,
+            predicates::bin_op_pred::BinOpType,
+            properties::{attr_ref, Attribute},
             values::Value,
         },
-        cost_model::tests::{
+        stats::DEFAULT_EQ_SEL,
+        test_utils::tests::MemoGroupInfo,
+        test_utils::tests::{
             attr_index, bin_op, cnst, create_four_table_mock_cost_model, create_mock_cost_model,
             create_three_table_mock_cost_model, create_two_table_mock_cost_model,
             create_two_table_mock_cost_model_custom_row_cnts, empty_per_attr_stats, log_op,
             per_attr_stats_with_dist_and_ndistinct, per_attr_stats_with_ndistinct,
-            TestOptCostModelMock, TestPerAttributeStats, TEST_ATTR1_NAME, TEST_ATTR2_NAME,
-            TEST_TABLE1_ID, TEST_TABLE2_ID, TEST_TABLE3_ID, TEST_TABLE4_ID,
+            TestOptCostModelMock, TEST_ATTR1_NAME, TEST_ATTR2_NAME, TEST_TABLE1_ID, TEST_TABLE2_ID,
+            TEST_TABLE3_ID, TEST_TABLE4_ID,
         },
-        memo_ext::tests::MemoGroupInfo,
-        stats::DEFAULT_EQ_SEL,
     };
 
     use super::*;
@@ -905,7 +904,8 @@ mod tests {
             expected_inner_sel
         );
         // check the outer sels
-        assert_outer_selectivities(&cost_model, expr_tree, expr_tree_rev, &attr_refs, 0.25, 0.2);
+        assert_outer_selectivities(&cost_model, expr_tree, expr_tree_rev, &attr_refs, 0.25, 0.2)
+            .await;
     }
 
     /// Non-unique oncond means the column is not unique in either table

optd-cost-model/src/cost/join/hash_join.rs

@@ -1,12 +1,5 @@
-use itertools::Itertools;
-
 use crate::{
-    common::{
-        nodes::{JoinType, ReprPredicateNode},
-        predicates::{attr_index_pred::AttrIndexPred, list_pred::ListPred},
-        properties::attr_ref::{AttrRefs, SemanticCorrelation},
-        types::GroupId,
-    },
+    common::{nodes::JoinType, predicates::list_pred::ListPred, types::GroupId},
     cost_model::CostModelImpl,
     storage::CostModelStorageManager,
     CostModelResult, EstimatedStatistic,
@@ -20,8 +13,8 @@ impl<S: CostModelStorageManager> CostModelImpl<S> {
         &self,
         join_typ: JoinType,
         group_id: GroupId,
-        left_row_cnt: f64,
-        right_row_cnt: f64,
+        left_row_cnt: EstimatedStatistic,
+        right_row_cnt: EstimatedStatistic,
         left_group_id: GroupId,
         right_group_id: GroupId,
         left_keys: ListPred,
@@ -42,14 +35,14 @@ impl<S: CostModelStorageManager> CostModelImpl<S> {
                 right_keys,
                 output_attr_refs.attr_refs(),
                 input_correlation,
-                left_row_cnt,
-                right_row_cnt,
+                left_row_cnt.0,
+                right_row_cnt.0,
                 left_attr_cnt,
             )
             .await?
         };
         Ok(EstimatedStatistic(
-            (left_row_cnt * right_row_cnt * selectivity).max(1.0),
+            (left_row_cnt.0 * right_row_cnt.0 * selectivity).max(1.0),
         ))
     }
 }

optd-cost-model/src/cost/join/nested_loop_join.rs

@@ -1,8 +1,6 @@
 use crate::{
     common::{
-        nodes::{ArcPredicateNode, JoinType, PredicateType, ReprPredicateNode},
-        predicates::log_op_pred::{LogOpPred, LogOpType},
-        properties::attr_ref::{AttrRefs, SemanticCorrelation},
+        nodes::{ArcPredicateNode, JoinType},
         types::GroupId,
     },
     cost_model::CostModelImpl,
@@ -18,8 +16,8 @@ impl<S: CostModelStorageManager> CostModelImpl<S> {
         &self,
         join_typ: JoinType,
         group_id: GroupId,
-        left_row_cnt: f64,
-        right_row_cnt: f64,
+        left_row_cnt: EstimatedStatistic,
+        right_row_cnt: EstimatedStatistic,
         left_group_id: GroupId,
         right_group_id: GroupId,
         join_cond: ArcPredicateNode,
@@ -36,13 +34,13 @@ impl<S: CostModelStorageManager> CostModelImpl<S> {
                 join_cond,
                 output_attr_refs.attr_refs(),
                 input_correlation,
-                left_row_cnt,
-                right_row_cnt,
+                left_row_cnt.0,
+                right_row_cnt.0,
             )
             .await?
         };
         Ok(EstimatedStatistic(
-            (left_row_cnt * right_row_cnt * selectivity).max(1.0),
+            (left_row_cnt.0 * right_row_cnt.0 * selectivity).max(1.0),
         ))
     }
 }

optd-cost-model/src/cost/mod.rs

@@ -1,5 +1,3 @@
-#![allow(unused)]
-
 pub mod agg;
 pub mod filter;
 pub mod join;