feat: hash cache for compaction and consolidate (#53)

* hash cache for compaction and consolidate

* use murmurhash

* changeset

* tweak

Author: samwillis

.changeset/thick-symbols-peel.md

@@ -0,0 +1,5 @@
+---
+'@electric-sql/d2ts': patch
+---
+
+better hashing in the consolidate and compaction methods

packages/d2ts/package.json

@@ -71,6 +71,8 @@
     }
   },
   "dependencies": {
-    "fractional-indexing": "^3.2.0"
+    "@types/murmurhash-js": "^1.0.6",
+    "fractional-indexing": "^3.2.0",
+    "murmurhash-js": "^1.0.0"
   }
 }

packages/d2ts/src/multiset.ts

@@ -1,4 +1,4 @@
-import { DefaultMap, chunkedArrayPush } from './utils.js'
+import { DefaultMap, chunkedArrayPush, hash } from './utils.js'
 
 export type MultiSetArray<T> = [T, number][]
 export type KeyedData<T> = [key: string, value: T]
@@ -67,6 +67,7 @@ export class MultiSet<T> {
    */
   consolidate(): MultiSet<T> {
     const consolidated = new DefaultMap<string | number, number>(() => 0)
+    const values = new Map<string, any>()
 
     let hasString = false
     let hasNumber = false
@@ -85,14 +86,17 @@ export class MultiSet<T> {
     const requireJson = hasOther || (hasString && hasNumber)
 
     for (const [data, multiplicity] of this.#inner) {
-      const key = requireJson ? JSON.stringify(data) : (data as string | number)
+      const key = requireJson ? hash(data) : (data as string | number)
+      if (requireJson && !values.has(key as string)) {
+        values.set(key as string, data)
+      }
       consolidated.update(key, (count) => count + multiplicity)
     }
 
     const result: MultiSetArray<T> = []
     for (const [key, multiplicity] of consolidated.entries()) {
       if (multiplicity !== 0) {
-        const parsedKey = requireJson ? JSON.parse(key as string) : key
+        const parsedKey = requireJson ? values.get(key as string) : key
         result.push([parsedKey as T, multiplicity])
       }
     }
@@ -253,10 +257,10 @@ export class MultiSet<T> {
    */
   distinct(): MultiSet<KeyedData<T>> {
     return this.reduce((vals: [T, number][]): [T, number][] => {
-      const consolidated = new Map<string, [T, number]>()
+      const consolidated = new Map<string | number, [T, number]>()
 
       for (const [val, multiplicity] of vals) {
-        const key = JSON.stringify(val)
+        const key = hash(val)
         const current = consolidated.get(key)?.[1] || 0
         consolidated.set(key, [val, current + multiplicity])
       }

packages/d2ts/src/utils.ts

@@ -1,3 +1,5 @@
+import murmurhash from 'murmurhash-js'
+
 /**
  * A map that uses WeakRefs to store objects, and automatically removes them when
  * they are no longer referenced.
@@ -67,3 +69,20 @@ export function chunkedArrayPush(array: unknown[], other: unknown[]) {
     }
   }
 }
+
+const hashCache = new WeakMap()
+
+/**
+ * A hash method that caches the hash of a value in a week map
+ */
+export function hash(data: any): string | number {
+  if (data === null || data === undefined || typeof data !== 'object') {
+    return JSON.stringify(data)
+  }
+
+  if (hashCache.has(data)) {
+    return hashCache.get(data)
+  }
+
+  return murmurhash.murmur3(JSON.stringify(JSON.stringify(data)))
+}

packages/d2ts/src/version-index.ts

@@ -1,6 +1,6 @@
 import { Version, Antichain } from './order.js'
 import { MultiSet } from './multiset.js'
-import { DefaultMap, chunkedArrayPush } from './utils.js'
+import { DefaultMap, chunkedArrayPush, hash } from './utils.js'
 
 type VersionMap<T> = DefaultMap<Version, T[]>
 type IndexMap<K, V> = DefaultMap<K, VersionMap<[V, number]>>
@@ -209,10 +209,10 @@ export class Index<K, V> implements IndexType<K, V> {
 
     const consolidateValues = (values: [V, number][]): [V, number][] => {
       // Use string representation of values as keys for proper deduplication
-      const consolidated = new Map<string, [V, number]>()
+      const consolidated = new Map<string | number, [V, number]>()
 
       for (const [value, multiplicity] of values) {
-        const key = JSON.stringify(value)
+        const key = hash(value)
         const existing = consolidated.get(key)
         if (existing) {
           consolidated.set(key, [value, existing[1] + multiplicity])