feat(query): add rule_deduplicated_sort (#18544)

* feat(query): add rule_deduplicated_sort

* fix code style

* fix code style

* fix

* fix

* fix test

Author: xiedeyantu

src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs

@@ -18,6 +18,7 @@ use databend_common_exception::Result;
 
 use crate::optimizer::optimizers::rule::RuleCommuteJoin;
 use crate::optimizer::optimizers::rule::RuleCommuteJoinBaseTable;
+use crate::optimizer::optimizers::rule::RuleDeduplicateSort;
 use crate::optimizer::optimizers::rule::RuleEagerAggregation;
 use crate::optimizer::optimizers::rule::RuleEliminateEvalScalar;
 use crate::optimizer::optimizers::rule::RuleEliminateFilter;
@@ -114,6 +115,7 @@ impl RuleFactory {
             RuleID::PushDownPrewhere => Ok(Box::new(RulePushDownPrewhere::new(metadata))),
             RuleID::TryApplyAggIndex => Ok(Box::new(RuleTryApplyAggIndex::new(metadata))),
             RuleID::EliminateSort => Ok(Box::new(RuleEliminateSort::new())),
+            RuleID::DeduplicateSort => Ok(Box::new(RuleDeduplicateSort::new())),
             RuleID::SemiToInnerJoin => Ok(Box::new(RuleSemiToInnerJoin::new())),
             RuleID::MergeFilterIntoMutation => {
                 Ok(Box::new(RuleMergeFilterIntoMutation::new(metadata)))

src/query/sql/src/planner/optimizer/optimizers/rule/rule.rs

@@ -27,6 +27,7 @@ use crate::optimizer::optimizers::rule::TransformResult;
 pub static DEFAULT_REWRITE_RULES: LazyLock<Vec<RuleID>> = LazyLock::new(|| {
     vec![
         RuleID::EliminateSort,
+        RuleID::DeduplicateSort,
         RuleID::EliminateUnion,
         RuleID::MergeEvalScalar,
         // Filter
@@ -113,6 +114,7 @@ pub enum RuleID {
     EliminateEvalScalar,
     EliminateFilter,
     EliminateSort,
+    DeduplicateSort,
     MergeEvalScalar,
     MergeFilter,
     GroupingSetsToUnion,
@@ -158,6 +160,7 @@ impl Display for RuleID {
             RuleID::EliminateEvalScalar => write!(f, "EliminateEvalScalar"),
             RuleID::EliminateFilter => write!(f, "EliminateFilter"),
             RuleID::EliminateSort => write!(f, "EliminateSort"),
+            RuleID::DeduplicateSort => write!(f, "DeduplicateSort"),
             RuleID::MergeEvalScalar => write!(f, "MergeEvalScalar"),
             RuleID::MergeFilter => write!(f, "MergeFilter"),
             RuleID::NormalizeScalarFilter => write!(f, "NormalizeScalarFilter"),

src/query/sql/src/planner/optimizer/optimizers/rule/sort_rules/mod.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod rule_deduplicate_sort;
 mod rule_eliminate_sort;
 
+pub use rule_deduplicate_sort::RuleDeduplicateSort;
 pub use rule_eliminate_sort::RuleEliminateSort;

src/query/sql/src/planner/optimizer/optimizers/rule/sort_rules/rule_deduplicate_sort.rs

@@ -0,0 +1,105 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use databend_common_exception::Result;
+
+use crate::optimizer::ir::Matcher;
+use crate::optimizer::ir::SExpr;
+use crate::optimizer::optimizers::rule::Rule;
+use crate::optimizer::optimizers::rule::RuleID;
+use crate::optimizer::optimizers::rule::TransformResult;
+use crate::plans::RelOp;
+use crate::plans::Sort;
+
+/// Rule to remove duplicate sort items in ORDER BY clause.
+///
+/// For example, transforms:
+/// ORDER BY x ASC NULLS LAST, x ASC NULLS LAST
+/// into:
+/// ORDER BY x ASC NULLS LAST
+///
+/// This optimization is valid because duplicate sort fields don't contribute
+/// additional ordering and only add unnecessary computational overhead.
+pub struct RuleDeduplicateSort {
+    id: RuleID,
+    matchers: Vec<Matcher>,
+}
+
+impl RuleDeduplicateSort {
+    pub fn new() -> Self {
+        Self {
+            id: RuleID::DeduplicateSort,
+            // Sort
+            //  \
+            //   *
+            matchers: vec![Matcher::MatchOp {
+                op_type: RelOp::Sort,
+                children: vec![Matcher::Leaf],
+            }],
+        }
+    }
+}
+
+impl Rule for RuleDeduplicateSort {
+    fn id(&self) -> RuleID {
+        self.id
+    }
+
+    fn apply(&self, s_expr: &SExpr, state: &mut TransformResult) -> Result<()> {
+        let sort: Sort = s_expr.plan().clone().try_into()?;
+
+        if sort.items.len() <= 1 {
+            return Ok(());
+        }
+
+        // Deduplicate sort items while preserving order
+        let mut deduplicated_items = Vec::with_capacity(sort.items.len());
+        let mut seen = std::collections::HashSet::with_capacity(sort.items.len());
+
+        for item in &sort.items {
+            if seen.insert(item.clone()) {
+                deduplicated_items.push(item.clone());
+            }
+        }
+
+        // Only apply transformation if we actually removed duplicates
+        if deduplicated_items.len() == sort.items.len() {
+            return Ok(());
+        }
+
+        let new_sort = Sort {
+            items: deduplicated_items,
+            limit: sort.limit,
+            after_exchange: sort.after_exchange,
+            pre_projection: sort.pre_projection,
+            window_partition: sort.window_partition,
+        };
+
+        let mut result = s_expr.replace_plan(std::sync::Arc::new(new_sort.into()));
+        result.set_applied_rule(&self.id);
+        state.add_result(result);
+
+        Ok(())
+    }
+
+    fn matchers(&self) -> &[Matcher] {
+        &self.matchers
+    }
+}
+
+impl Default for RuleDeduplicateSort {
+    fn default() -> Self {
+        Self::new()
+    }
+}

tests/sqllogictests/suites/mode/standalone/explain/eliminate_sort.test

@@ -58,7 +58,7 @@ explain select * from (select * from numbers(10) t order by t.number desc) order
 ----
 Sort(Single)
 ├── output columns: [t.number (#0)]
-├── sort keys: [number DESC NULLS LAST, number DESC NULLS LAST]
+├── sort keys: [number DESC NULLS LAST]
 ├── estimated rows: 10.00
 └── Sort(Single)
     ├── output columns: [t.number (#0)]

tests/sqllogictests/suites/mode/standalone/explain/explain_deduplicate_sort.test

@@ -0,0 +1,24 @@
+statement ok
+CREATE TABLE emp (deptno INT, job STRING, sal INT);
+
+query T
+EXPLAIN SELECT deptno AS d, job, sal 
+FROM emp 
+ORDER BY deptno, d, job ASC, job DESC, sal NULLS FIRST, sal NULLS LAST, deptno;
+----
+Sort(Single)
+├── output columns: [emp.deptno (#0), emp.job (#1), emp.sal (#2)]
+├── sort keys: [deptno ASC NULLS LAST, job ASC NULLS LAST, job DESC NULLS LAST, sal ASC NULLS FIRST, sal ASC NULLS LAST]
+├── estimated rows: 0.00
+└── TableScan
+    ├── table: default.default.emp
+    ├── output columns: [deptno (#0), job (#1), sal (#2)]
+    ├── read rows: 0
+    ├── read size: 0
+    ├── partitions total: 0
+    ├── partitions scanned: 0
+    ├── push downs: [filters: [], limit: NONE]
+    └── estimated rows: 0.00
+
+statement ok
+DROP TABLE emp;