automatically create indexes when a collection is queried (#292)

Author: samwillis

.changeset/petite-bars-lay.md

@@ -0,0 +1,5 @@
+---
+"@tanstack/db": patch
+---
+
+Added an auto-indexing system that creates indexes on collection eagerly when querying, this is a performance optimization that can be disabled by setting the autoIndex option to `off`.

packages/db/src/collection.ts

@@ -6,6 +6,7 @@ import {
 } from "./query/builder/ref-proxy"
 import { OrderedIndex } from "./indexes/ordered-index.js"
 import { IndexProxy, LazyIndexWrapper } from "./indexes/lazy-index.js"
+import { ensureIndexForExpression } from "./indexes/auto-index.js"
 import { createTransaction, getActiveTransaction } from "./transactions"
 import { createFilteredCallback, currentStateAsChanges } from "./change-events"
 import type { Transaction } from "./transactions"
@@ -426,7 +427,11 @@ export class CollectionImpl<
       a.compareCreatedAt(b)
     )
 
-    this.config = config
+    // Set default values for optional config properties
+    this.config = {
+      ...config,
+      autoIndex: config.autoIndex ?? `eager`,
+    }
 
     // Store in global collections store
     collectionsStore.set(this.id, this)
@@ -1359,12 +1364,18 @@ export class CollectionImpl<
 
     this.lazyIndexes.set(indexId, lazyIndex)
 
-    // For synchronous constructors (classes), resolve immediately
-    // For async loaders, wait for collection to be ready
-    if (typeof resolver === `function` && resolver.prototype) {
-      // This is a constructor - resolve immediately and synchronously
+    // For OrderedIndex, resolve immediately and synchronously
+    if ((resolver as unknown) === OrderedIndex) {
+      try {
+        const resolvedIndex = lazyIndex.getResolved()
+        this.resolvedIndexes.set(indexId, resolvedIndex)
+      } catch (error) {
+        console.warn(`Failed to resolve OrderedIndex:`, error)
+      }
+    } else if (typeof resolver === `function` && resolver.prototype) {
+      // Other synchronous constructors - resolve immediately
       try {
-        const resolvedIndex = lazyIndex.getResolved() // This should work since constructor resolved it
+        const resolvedIndex = lazyIndex.getResolved()
         this.resolvedIndexes.set(indexId, resolvedIndex)
       } catch {
         // Fallback to async resolution
@@ -2160,6 +2171,11 @@ export class CollectionImpl<
     // Start sync and track subscriber
     this.addSubscriber()
 
+    // Auto-index for where expressions if enabled
+    if (options.whereExpression) {
+      ensureIndexForExpression(options.whereExpression, this)
+    }
+
     // Create a filtered callback if where clause is provided
     const filteredCallback =
       options.where || options.whereExpression

packages/db/src/indexes/auto-index.ts

@@ -0,0 +1,108 @@
+import { OrderedIndex } from "./ordered-index"
+import type { BasicExpression } from "../query/ir"
+import type { CollectionImpl } from "../collection"
+
+export interface AutoIndexConfig {
+  autoIndex?: `off` | `eager`
+}
+
+/**
+ * Analyzes a where expression and creates indexes for all simple operations on single fields
+ */
+export function ensureIndexForExpression<
+  T extends Record<string, any>,
+  TKey extends string | number,
+>(
+  expression: BasicExpression,
+  collection: CollectionImpl<T, TKey, any, any, any>
+): void {
+  // Only proceed if auto-indexing is enabled
+  if (collection.config.autoIndex !== `eager`) {
+    return
+  }
+
+  // Don't auto-index during sync operations
+  if (
+    collection.status === `loading` ||
+    collection.status === `initialCommit`
+  ) {
+    return
+  }
+
+  // Extract all indexable expressions and create indexes for them
+  const indexableExpressions = extractIndexableExpressions(expression)
+
+  for (const { fieldName, fieldPath } of indexableExpressions) {
+    // Check if we already have an index for this field
+    const existingIndex = Array.from(collection.indexes.values()).find(
+      (index) => index.matchesField(fieldPath)
+    )
+
+    if (existingIndex) {
+      continue // Index already exists
+    }
+
+    // Create a new index for this field using the collection's createIndex method
+    try {
+      collection.createIndex((row) => (row as any)[fieldName], {
+        name: `auto_${fieldName}`,
+        indexType: OrderedIndex,
+      })
+    } catch (error) {
+      console.warn(
+        `Failed to create auto-index for field "${fieldName}":`,
+        error
+      )
+    }
+  }
+}
+
+/**
+ * Extracts all indexable expressions from a where expression
+ */
+function extractIndexableExpressions(
+  expression: BasicExpression
+): Array<{ fieldName: string; fieldPath: Array<string> }> {
+  const results: Array<{ fieldName: string; fieldPath: Array<string> }> = []
+
+  function extractFromExpression(expr: BasicExpression): void {
+    if (expr.type !== `func`) {
+      return
+    }
+
+    const func = expr as any
+
+    // Handle 'and' expressions by recursively processing all arguments
+    if (func.name === `and`) {
+      for (const arg of func.args) {
+        extractFromExpression(arg)
+      }
+      return
+    }
+
+    // Check if this is a supported operation
+    const supportedOperations = [`eq`, `gt`, `gte`, `lt`, `lte`, `in`]
+    if (!supportedOperations.includes(func.name)) {
+      return
+    }
+
+    // Check if the first argument is a property reference (single field)
+    if (func.args.length < 1 || func.args[0].type !== `ref`) {
+      return
+    }
+
+    const fieldRef = func.args[0]
+    const fieldPath = fieldRef.path
+
+    // Skip if it's not a simple field (e.g., nested properties or array access)
+    if (fieldPath.length !== 1) {
+      return
+    }
+
+    const fieldName = fieldPath[0]
+    results.push({ fieldName, fieldPath })
+  }
+
+  extractFromExpression(expression)
+  return results
+}

packages/db/src/types.ts

@@ -377,6 +377,15 @@ export interface CollectionConfig<
    * Defaults to false for lazy loading. Set to true to immediately sync.
    */
   startSync?: boolean
+  /**
+   * Auto-indexing mode for the collection.
+   * When enabled, indexes will be automatically created for simple where expressions.
+   * @default "eager"
+   * @description
+   * - "off": No automatic indexing
+   * - "eager": Automatically create indexes for simple where expressions in subscribeChanges (default)
+   */
+  autoIndex?: `off` | `eager`
   /**
    * Optional function to compare two items.
    * This is used to order the items in the collection.