Incremental topK with fractional index

.changeset/chilly-icons-design.md

@@ -0,0 +1,5 @@
+---
+'@electric-sql/d2mini': patch
+---
+
+Introduce topKWithFractionalIndexBTree and orderByWithFractionalIndexBTree operators. These variants use a B+ tree which should be efficient for big collections (logarithmic time).

packages/d2mini/package.json

@@ -50,6 +50,7 @@
   },
   "dependencies": {
     "fractional-indexing": "^3.2.0",
-    "murmurhash-js": "^1.0.0"
+    "murmurhash-js": "^1.0.0",
+    "sorted-btree": "^1.8.1"
   }
 }

packages/d2mini/src/indexes.ts

@@ -35,6 +35,13 @@ export class Index<K, V> {
     return [...valueMap.values()]
   }
 
+  getMultiplicity(key: K, value: V): number {
+    const valueMap = this.#inner.get(key)
+    const valueHash = hash(value)
+    const [, multiplicity] = valueMap.get(valueHash)
+    return multiplicity
+  }
+
   entries() {
     return this.#inner.entries()
   }

packages/d2mini/src/operators/orderBy.ts

@@ -2,6 +2,7 @@ import { IStreamBuilder } from '../types'
 import { KeyValue } from '../types.js'
 import { topK, topKWithIndex } from './topK.js'
 import { topKWithFractionalIndex } from './topKWithFractionalIndex.js'
+import { topKWithFractionalIndexBTree } from './topKWithFractionalIndexBTree.js'
 import { map } from './map.js'
 import { innerJoin } from './join.js'
 import { consolidate } from './consolidate.js'
@@ -128,19 +129,11 @@ export function orderByWithIndex<
   }
 }
 
-/**
- * Orders the elements and limits the number of results, with optional offset and
- * annotates the value with a fractional index.
- * This requires a keyed stream, and uses the `topKWithFractionalIndex` operator to order all the elements.
- *
- * @param valueExtractor - A function that extracts the value to order by from the element
- * @param options - An optional object containing comparator, limit and offset properties
- * @returns A piped operator that orders the elements and limits the number of results
- */
-export function orderByWithFractionalIndex<
+function orderByWithFractionalIndexBase<
   T extends KeyValue<unknown, unknown>,
   Ve = unknown,
 >(
+  topK: typeof topKWithFractionalIndex,
   valueExtractor: (
     value: T extends KeyValue<unknown, infer V> ? V : never,
   ) => Ve,
@@ -181,7 +174,7 @@ export function orderByWithFractionalIndex<
             ],
           ] as KeyValue<null, [K, Ve]>,
       ),
-      topKWithFractionalIndex((a, b) => comparator(a[1], b[1]), {
+      topK((a, b) => comparator(a[1], b[1]), {
         limit,
         offset,
       }),
@@ -194,3 +187,44 @@ export function orderByWithFractionalIndex<
     )
   }
 }
+
+/**
+ * Orders the elements and limits the number of results, with optional offset and
+ * annotates the value with a fractional index.
+ * This requires a keyed stream, and uses the `topKWithFractionalIndex` operator to order all the elements.
+ *
+ * @param valueExtractor - A function that extracts the value to order by from the element
+ * @param options - An optional object containing comparator, limit and offset properties
+ * @returns A piped operator that orders the elements and limits the number of results
+ */
+export function orderByWithFractionalIndex<
+  T extends KeyValue<unknown, unknown>,
+  Ve = unknown,
+>(
+  valueExtractor: (
+    value: T extends KeyValue<unknown, infer V> ? V : never,
+  ) => Ve,
+  options?: OrderByOptions<Ve>,
+) {
+  return orderByWithFractionalIndexBase(
+    topKWithFractionalIndex,
+    valueExtractor,
+    options,
+  )
+}
+
+export function orderByWithFractionalIndexBTree<
+  T extends KeyValue<unknown, unknown>,
+  Ve = unknown,
+>(
+  valueExtractor: (
+    value: T extends KeyValue<unknown, infer V> ? V : never,
+  ) => Ve,
+  options?: OrderByOptions<Ve>,
+) {
+  return orderByWithFractionalIndexBase(
+    topKWithFractionalIndexBTree,
+    valueExtractor,
+    options,
+  )
+}