Feat/Index Cover (#296)

* feat: support index cover & fix is_null can't use index

* chore: add unit test for index cover & fix string_encode and string_decode

* chore: add test for composite index trailing columns cover

* chore: fix null_first in sort

Author: KKould

Makefile

@@ -0,0 +1,25 @@
+# Helper variables (override on invocation if needed).
+CARGO ?= cargo
+WASM_PACK ?= wasm-pack
+SQLLOGIC_PATH ?= tests/slt/**/*.slt
+
+.PHONY: test test-wasm test-slt test-all wasm-build
+
+## Run default Rust tests in the current environment (non-WASM).
+test:
+	$(CARGO) test
+
+## Build the WebAssembly package (artifact goes to ./pkg).
+wasm-build:
+	$(WASM_PACK) build --release --target nodejs
+
+## Execute wasm-bindgen tests under Node.js (wasm32 target).
+test-wasm:
+	$(WASM_PACK) test --node --release
+
+## Run the sqllogictest harness against the configured .slt suite.
+test-slt:
+	$(CARGO) run -p sqllogictest-test -- --path $(SQLLOGIC_PATH)
+
+## Convenience target to run every suite in sequence.
+test-all: test test-wasm test-slt

src/execution/dql/index_scan.rs

@@ -4,15 +4,31 @@ use crate::planner::operator::table_scan::TableScanOperator;
 use crate::storage::{Iter, StatisticsMetaCache, TableCache, Transaction, ViewCache};
 use crate::throw;
 use crate::types::index::IndexMetaRef;
+use crate::types::serialize::TupleValueSerializableImpl;
 
 pub(crate) struct IndexScan {
     op: TableScanOperator,
     index_by: IndexMetaRef,
     ranges: Vec<Range>,
+    covered_deserializers: Option<Vec<TupleValueSerializableImpl>>,
 }
 
-impl From<(TableScanOperator, IndexMetaRef, Range)> for IndexScan {
-    fn from((op, index_by, range): (TableScanOperator, IndexMetaRef, Range)) -> Self {
+impl
+    From<(
+        TableScanOperator,
+        IndexMetaRef,
+        Range,
+        Option<Vec<TupleValueSerializableImpl>>,
+    )> for IndexScan
+{
+    fn from(
+        (op, index_by, range, covered_deserializers): (
+            TableScanOperator,
+            IndexMetaRef,
+            Range,
+            Option<Vec<TupleValueSerializableImpl>>,
+        ),
+    ) -> Self {
         let ranges = match range {
             Range::SortedRanges(ranges) => ranges,
             range => vec![range],
@@ -22,6 +38,7 @@ impl From<(TableScanOperator, IndexMetaRef, Range)> for IndexScan {
             op,
             index_by,
             ranges,
+            covered_deserializers,
         }
     }
 }
@@ -51,6 +68,7 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for IndexScan {
                     self.index_by,
                     self.ranges,
                     with_pk,
+                    self.covered_deserializers,
                 )
             );
 

src/execution/dql/sort.rs

@@ -183,13 +183,13 @@ impl SortBy {
                         let mut key = BumpBytes::new_in(arena);
 
                         expr.eval(Some((tuple, schema)))?
-                            .memcomparable_encode(&mut key)?;
-                        if !asc {
-                            for byte in key.iter_mut() {
+                            .memcomparable_encode_with_null_order(&mut key, *nulls_first)?;
+
+                        if !asc && key.len() > 1 {
+                            for byte in key.iter_mut().skip(1) {
                                 *byte ^= 0xFF;
                             }
                         }
-                        key.push(if *nulls_first { u8::MIN } else { u8::MAX });
                         full_key.extend(key);
                     }
                     sort_keys.put((i, full_key))

src/execution/dql/top_k.rs

@@ -48,13 +48,12 @@ fn top_sort<'a>(
     {
         let mut key = BumpBytes::new_in(arena);
         expr.eval(Some((&tuple, &**schema)))?
-            .memcomparable_encode(&mut key)?;
-        if !asc {
-            for byte in key.iter_mut() {
+            .memcomparable_encode_with_null_order(&mut key, *nulls_first)?;
+        if !asc && key.len() > 1 {
+            for byte in key.iter_mut().skip(1) {
                 *byte ^= 0xFF;
             }
         }
-        key.push(if *nulls_first { u8::MIN } else { u8::MAX });
         full_key.extend(key);
     }
 

src/execution/mod.rs

@@ -124,9 +124,11 @@ pub fn build_read<'a, T: Transaction + 'a>(
             if let Some(PhysicalOption::IndexScan(IndexInfo {
                 meta,
                 range: Some(range),
+                covered_deserializers,
             })) = plan.physical_option
             {
-                IndexScan::from((op, meta, range)).execute(cache, transaction)
+                IndexScan::from((op, meta, range, covered_deserializers))
+                    .execute(cache, transaction)
             } else {
                 SeqScan::from(op).execute(cache, transaction)
             }

src/optimizer/core/memo.rs

@@ -212,6 +212,7 @@ mod tests {
                         max: Bound::Unbounded,
                     }
                 ])),
+                covered_deserializers: None,
             }))
         );
 

src/optimizer/rule/implementation/dql/table_scan.rs

@@ -79,7 +79,9 @@ impl<T: Transaction> ImplementationRule<T> for IndexScanImplementation {
                     {
                         let mut row_count = statistics_meta.collect_count(range)?;
 
-                        if !matches!(index_info.meta.ty, IndexType::PrimaryKey { .. }) {
+                        if index_info.covered_deserializers.is_none()
+                            && !matches!(index_info.meta.ty, IndexType::PrimaryKey { .. })
+                        {
                             // need to return table query(non-covering index)
                             row_count *= 2;
                         }

src/optimizer/rule/normalization/pushdown_predicates.rs

@@ -13,9 +13,9 @@ use crate::types::index::{IndexInfo, IndexMetaRef, IndexType};
 use crate::types::value::DataValue;
 use crate::types::LogicalType;
 use itertools::Itertools;
-use std::mem;
 use std::ops::Bound;
 use std::sync::LazyLock;
+use std::{mem, slice};
 
 static PUSH_PREDICATE_THROUGH_JOIN: LazyLock<Pattern> = LazyLock::new(|| Pattern {
     predicate: |op| matches!(op, Operator::Filter(_)),
@@ -215,12 +215,17 @@ impl NormalizationRule for PushPredicateIntoScan {
     fn apply(&self, node_id: HepNodeId, graph: &mut HepGraph) -> Result<(), DatabaseError> {
         if let Operator::Filter(op) = graph.operator(node_id).clone() {
             if let Some(child_id) = graph.eldest_child_at(node_id) {
-                if let Operator::TableScan(child_op) = graph.operator_mut(child_id) {
-                    //FIXME: now only support `unique` and `primary key`
-                    for IndexInfo { meta, range } in &mut child_op.index_infos {
+                if let Operator::TableScan(scan_op) = graph.operator_mut(child_id) {
+                    for IndexInfo {
+                        meta,
+                        range,
+                        covered_deserializers,
+                    } in &mut scan_op.index_infos
+                    {
                         if range.is_some() {
                             continue;
                         }
+                        // range detach
                         *range = match meta.ty {
                             IndexType::PrimaryKey { is_multiple: false }
                             | IndexType::Unique
@@ -232,6 +237,28 @@ impl NormalizationRule for PushPredicateIntoScan {
                                 Self::composite_range(&op, meta)?
                             }
                         };
+                        // try index covered
+                        let mut deserializers = Vec::with_capacity(meta.column_ids.len());
+                        let mut cover_count = 0;
+                        let index_column_types = match &meta.value_ty {
+                            LogicalType::Tuple(tys) => tys,
+                            ty => slice::from_ref(ty),
+                        };
+                        for (i, column_id) in meta.column_ids.iter().enumerate() {
+                            for column in scan_op.columns.values() {
+                                deserializers.push(
+                                    if column.id().map(|id| &id == column_id).unwrap_or(false) {
+                                        cover_count += 1;
+                                        column.datatype().serializable()
+                                    } else {
+                                        index_column_types[i].skip_serializable()
+                                    },
+                                );
+                            }
+                        }
+                        if cover_count == scan_op.columns.len() {
+                            *covered_deserializers = Some(deserializers)
+                        }
                     }
                 }
             }

src/planner/operator/table_scan.rs

@@ -48,6 +48,7 @@ impl TableScanOperator {
             .map(|meta| IndexInfo {
                 meta: meta.clone(),
                 range: None,
+                covered_deserializers: None,
             })
             .collect_vec();
 

src/storage/memory.rs

@@ -218,6 +218,7 @@ mod wasm_tests {
                 max: Bound::Included(DataValue::Int32(2)),
             }],
             true,
+            None,
         )?;
 
         let mut result = Vec::new();
@@ -349,6 +350,7 @@ mod native_tests {
                 max: Bound::Included(DataValue::Int32(2)),
             }],
             true,
+            None,
         )?;
 
         let mut result = Vec::new();