feat(optimizer): add optimizer skip list (#17750)

Author: BohuTANG

src/query/settings/src/settings_default.rs

@@ -1249,6 +1249,14 @@ impl DefaultSettings {
                     scope: SettingScope::Both,
                     range: Some(SettingRange::Numeric(0..=1)),
                 }),
+                // Add this to the HashMap in DefaultSettings::instance()
+   ("optimizer_skip_list", DefaultSettingValue {
+                value: UserSettingValue::String(String::new()),
+                desc: "Comma-separated(,) list of optimizer names to skip during query optimization",
+                mode: SettingMode::Both,
+                scope: SettingScope::Both,
+                range: None,
+            }),
 
             ]);
 

src/query/settings/src/settings_getter_setter.rs

@@ -928,4 +928,8 @@ impl Settings {
     pub fn get_enable_optimizer_trace(&self) -> Result<bool> {
         Ok(self.try_get_u64("enable_optimizer_trace")? == 1)
     }
+
+    pub fn get_optimizer_skip_list(&self) -> Result<String> {
+        self.try_get_string("optimizer_skip_list")
+    }
 }

src/query/sql/src/planner/optimizer/optimizer.rs

@@ -36,7 +36,7 @@ use crate::optimizer::optimizers::recursive::RecursiveRuleOptimizer;
 use crate::optimizer::optimizers::rule::RuleID;
 use crate::optimizer::optimizers::rule::DEFAULT_REWRITE_RULES;
 use crate::optimizer::optimizers::CascadesOptimizer;
-use crate::optimizer::optimizers::DPhpy;
+use crate::optimizer::optimizers::DPhpyOptimizer;
 use crate::optimizer::pipeline::OptimizerPipeline;
 use crate::optimizer::statistics::CollectStatisticsOptimizer;
 use crate::optimizer::OptimizerContext;
@@ -54,7 +54,7 @@ use crate::plans::SetScalarsOrQuery;
 use crate::InsertInputSource;
 
 #[fastrace::trace]
-#[async_recursion(#[recursive::recursive])]
+#[async_recursion(# [recursive::recursive])]
 pub async fn optimize(opt_ctx: Arc<OptimizerContext>, plan: Plan) -> Result<Plan> {
     match plan {
         Plan::Query {
@@ -262,7 +262,7 @@ pub async fn optimize_query(opt_ctx: Arc<OptimizerContext>, s_expr: SExpr) -> Re
             RuleID::SplitAggregate,
         ]))
         // 9. Apply DPhyp algorithm for cost-based join reordering
-        .add(DPhpy::new(opt_ctx.clone()))
+        .add(DPhpyOptimizer::new(opt_ctx.clone()))
         // 10. After join reorder, Convert some single join to inner join.
         .add(SingleToInnerOptimizer::new())
         // 11. Deduplicate join conditions.
@@ -306,7 +306,7 @@ async fn get_optimized_memo(opt_ctx: Arc<OptimizerContext>, s_expr: SExpr) -> Re
             RuleID::SplitAggregate,
         ]))
         // Cost based optimization
-        .add(DPhpy::new(opt_ctx.clone()))
+        .add(DPhpyOptimizer::new(opt_ctx.clone()))
         .add(CascadesOptimizer::new(opt_ctx.clone())?);
 
     let _s_expr = pipeline.execute().await?;

src/query/sql/src/planner/optimizer/optimizer_api.rs

@@ -18,15 +18,14 @@ use crate::optimizer::ir::Memo;
 use crate::optimizer::ir::SExpr;
 
 /// Trait defining the interface for query optimizers.
-
 #[async_trait::async_trait]
 pub trait Optimizer: Send + Sync {
-    /// Optimize the given expression and return the optimized version.
-    async fn optimize(&mut self, expr: &SExpr) -> Result<SExpr>;
-
     /// Returns a unique identifier for this optimizer.
     fn name(&self) -> String;
 
+    /// Optimize the given expression and return the optimized version.
+    async fn optimize(&mut self, expr: &SExpr) -> Result<SExpr>;
+
     /// Get the memo if this optimizer maintains one.
     /// Default implementation returns None for optimizers that don't use a memo.
     fn memo(&self) -> Option<&Memo> {

src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs

@@ -27,8 +27,8 @@ use crate::optimizer::ir::SExpr;
 use crate::optimizer::optimizers::cascades::cost::DefaultCostModel;
 use crate::optimizer::optimizers::cascades::rule::StrategyFactory;
 use crate::optimizer::optimizers::cascades::tasks::OptimizeGroupTask;
-use crate::optimizer::optimizers::cascades::tasks::Scheduler;
 use crate::optimizer::optimizers::cascades::tasks::Task;
+use crate::optimizer::optimizers::cascades::tasks::TaskManager;
 use crate::optimizer::optimizers::cascades::tasks::DEFAULT_TASK_LIMIT;
 use crate::optimizer::optimizers::distributed::DistributedOptimizer;
 use crate::optimizer::optimizers::distributed::SortAndLimitPushDownOptimizer;
@@ -178,7 +178,7 @@ impl CascadesOptimizer {
             0
         };
 
-        let mut scheduler = Scheduler::new().with_task_limit(task_limit);
+        let mut scheduler = TaskManager::new().with_task_limit(task_limit);
         scheduler.add_task(Task::OptimizeGroup(root_task));
         scheduler.run(self)?;
 

src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_expr.rs

@@ -22,8 +22,8 @@ use educe::Educe;
 use super::apply_rule::ApplyRuleTask;
 use super::explore_group::ExploreGroupTask;
 use super::Task;
-use crate::optimizer::optimizers::cascades::tasks::Scheduler;
 use crate::optimizer::optimizers::cascades::tasks::SharedCounter;
+use crate::optimizer::optimizers::cascades::tasks::TaskManager;
 use crate::optimizer::optimizers::cascades::CascadesOptimizer;
 use crate::IndexType;
 
@@ -82,7 +82,7 @@ impl ExploreExprTask {
     pub fn execute(
         mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<Option<Task>> {
         if matches!(self.state, ExploreExprState::ExploredSelf) {
             return Ok(None);
@@ -94,7 +94,7 @@ impl ExploreExprTask {
     fn transition(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<()> {
         let event = match self.state {
             ExploreExprState::Init => self.explore_children(optimizer, scheduler)?,
@@ -124,7 +124,7 @@ impl ExploreExprTask {
     fn explore_children(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<ExploreExprEvent> {
         let m_expr = optimizer
             .memo
@@ -152,7 +152,7 @@ impl ExploreExprTask {
     fn explore_self(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<ExploreExprEvent> {
         let m_expr = optimizer
             .memo

src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/explore_group.rs

@@ -22,8 +22,8 @@ use educe::Educe;
 use super::explore_expr::ExploreExprTask;
 use super::Task;
 use crate::optimizer::ir::GroupState;
-use crate::optimizer::optimizers::cascades::tasks::Scheduler;
 use crate::optimizer::optimizers::cascades::tasks::SharedCounter;
+use crate::optimizer::optimizers::cascades::tasks::TaskManager;
 use crate::optimizer::optimizers::cascades::CascadesOptimizer;
 use crate::IndexType;
 
@@ -74,7 +74,7 @@ impl ExploreGroupTask {
     pub fn execute(
         mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<Option<Task>> {
         if matches!(self.state, ExploreGroupState::Explored) {
             return Ok(None);
@@ -86,7 +86,7 @@ impl ExploreGroupTask {
     pub fn transition(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<()> {
         let event = match self.state {
             ExploreGroupState::Init => self.explore_group(optimizer, scheduler)?,
@@ -111,7 +111,7 @@ impl ExploreGroupTask {
     fn explore_group(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<ExploreGroupEvent> {
         let group = optimizer.memo.group_mut(self.group_index)?;
 

src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/mod.rs

@@ -27,5 +27,5 @@ pub use optimize_expr::OptimizeExprTask;
 pub use optimize_group::OptimizeGroupTask;
 pub use task::SharedCounter;
 pub use task::Task;
-pub use task_manager::Scheduler;
+pub use task_manager::TaskManager;
 pub use task_manager::DEFAULT_TASK_LIMIT;

src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_expr.rs

@@ -31,8 +31,8 @@ use crate::optimizer::ir::PatternExtractor;
 use crate::optimizer::ir::RelExpr;
 use crate::optimizer::ir::RequiredProperty;
 use crate::optimizer::ir::SExpr;
-use crate::optimizer::optimizers::cascades::tasks::Scheduler;
 use crate::optimizer::optimizers::cascades::tasks::SharedCounter;
+use crate::optimizer::optimizers::cascades::tasks::TaskManager;
 use crate::optimizer::optimizers::cascades::CascadesOptimizer;
 use crate::plans::RelOperator;
 use crate::IndexType;
@@ -117,7 +117,7 @@ impl OptimizeExprTask {
     pub fn execute(
         mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<Option<Task>> {
         if matches!(self.state, OptimizeExprState::Finished) {
             return Ok(None);
@@ -129,7 +129,7 @@ impl OptimizeExprTask {
     fn transition(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<()> {
         let event = match self.state {
             OptimizeExprState::Init => OptimizeExprEvent::OptimizingChildren,
@@ -181,7 +181,7 @@ impl OptimizeExprTask {
     fn optimize_children(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<OptimizeExprEvent> {
         let m_expr = optimizer
             .memo
@@ -268,7 +268,7 @@ impl OptimizeExprTask {
     fn optimize_self(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        _scheduler: &mut Scheduler,
+        _scheduler: &mut TaskManager,
     ) -> Result<OptimizeExprEvent> {
         let m_expr = optimizer
             .memo

src/query/sql/src/planner/optimizer/optimizers/cascades/tasks/optimize_group.rs

@@ -24,8 +24,8 @@ use super::Task;
 use crate::optimizer::ir::RelExpr;
 use crate::optimizer::ir::RequiredProperty;
 use crate::optimizer::optimizers::cascades::tasks::ExploreGroupTask;
-use crate::optimizer::optimizers::cascades::tasks::Scheduler;
 use crate::optimizer::optimizers::cascades::tasks::SharedCounter;
+use crate::optimizer::optimizers::cascades::tasks::TaskManager;
 use crate::optimizer::optimizers::cascades::CascadesOptimizer;
 use crate::plans::Operator;
 use crate::IndexType;
@@ -90,7 +90,7 @@ impl OptimizeGroupTask {
     pub fn execute(
         mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<Option<Task>> {
         if matches!(self.state, OptimizeGroupState::Optimized) {
             return Ok(None);
@@ -102,7 +102,7 @@ impl OptimizeGroupTask {
     fn transition(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<()> {
         let event = match self.state {
             OptimizeGroupState::Init => self.explore_group(optimizer, scheduler)?,
@@ -133,7 +133,7 @@ impl OptimizeGroupTask {
     fn explore_group(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<OptimizeGroupEvent> {
         let group = optimizer.memo.group(self.group_index)?;
         if !group.state.explored() {
@@ -149,7 +149,7 @@ impl OptimizeGroupTask {
     fn optimize_group(
         &mut self,
         optimizer: &mut CascadesOptimizer,
-        scheduler: &mut Scheduler,
+        scheduler: &mut TaskManager,
     ) -> Result<OptimizeGroupEvent> {
         let group = optimizer.memo.group_mut(self.group_index)?;