support filter on dist func in Reader (#23183)

- implement filter by distance in Reader for IVFFlat
- replace per-block heaps by universal heap to reduce copy
- refactor some interfaces

Approved by: @zhangxu19830126, @heni02, @LeftHandCold, @XuPeng-SH, @fengttt, @ouyuanning

Author: aunjgr

pkg/defines/type.go

@@ -253,6 +253,10 @@ type InSp struct{}
 // This key is set on context when invoking internal SQL from ivf_search.
 type IvfBloomFilter struct{}
 
+// IvfReaderParam carries DistRange for ivf entries scan in internal SQL executor.
+// This key is set on context when invoking internal SQL from ivf_search.
+type IvfReaderParam struct{}
+
 // PkCheckByTN whether TN does primary key uniqueness check against transaction's workspace or not.
 type PkCheckByTN struct{}
 

pkg/frontend/test/engine_mock.go

@@ -21,6 +21,7 @@ import (
 	"github.com/matrixorigin/matrixone/pkg/container/batch"
 	"github.com/matrixorigin/matrixone/pkg/container/types"
 	"github.com/matrixorigin/matrixone/pkg/fileservice"
+	"github.com/matrixorigin/matrixone/pkg/pb/plan"
 	"github.com/matrixorigin/matrixone/pkg/vectorindex/metric"
 	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
 	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index"
@@ -64,12 +65,37 @@ func (f BlockReadFilter) DecideSearchFunc(isSortedBlk bool) ReadFilterSearchFunc
 	return nil
 }
 
-type BlockReadTopOp struct {
-	Typ    types.T
-	Metric metric.MetricType
-	ColPos int32
-	NumVec []byte
-	Limit  uint64
+type Float64Heap []float64
+
+func (h Float64Heap) Len() int           { return len(h) }
+func (h Float64Heap) Less(i, j int) bool { return h[i] > h[j] }
+func (h Float64Heap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *Float64Heap) Push(x any) {
+	*h = append(*h, x.(float64))
+}
+
+func (h *Float64Heap) Pop() any {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}
+
+type IndexReaderTopOp struct {
+	Typ        types.T
+	MetricType metric.MetricType
+	ColPos     int32
+	NumVec     []byte
+	Limit      uint64
+
+	LowerBoundType plan.BoundType
+	UpperBoundType plan.BoundType
+	LowerBound     float64
+	UpperBound     float64
+
+	DistHeap Float64Heap
 }
 
 type WriteOptions struct {