Switch the staged index syntax to {staged:true} (#40020)

Add tests to ensure they don't show up in the table definitions.

GitOrigin-RevId: 45088d58fc689e2ad0dad96bb7fcca53d62f64fe

Author: nipunn1313

npm-packages/convex/src/server/schema.test.ts

@@ -453,6 +453,96 @@ describe("DataModelFromSchemaDefinition", () => {
     };
     assert<Equals<DataModel, ExpectedDataModel>>();
   });
+
+  test("defineSchema creates staged indexes", () => {
+    const schema = defineSchema({
+      table: defineTable({
+        enabled: v.string(),
+        enabled2: v.string(),
+        staged: v.string(),
+      })
+        .index("by_enabled", ["enabled"])
+        .index("by_enabled2", ["enabled2"], { staged: false })
+        .index("by_staged", ["staged"], { staged: true }),
+    });
+    type Indexes = DataModelFromSchemaDefinition<
+      typeof schema
+    >["table"]["indexes"];
+    type ExpectedIndexes = {
+      by_enabled: ["enabled", "_creationTime"];
+      by_enabled2: ["enabled2", "_creationTime"];
+      by_id: ["_id"];
+      by_creation_time: ["_creationTime"];
+    };
+    assert<Equals<Indexes, ExpectedIndexes>>();
+  });
+
+  test("defineSchema creates staged search indexes", () => {
+    const schema = defineSchema({
+      table: defineTable({
+        enabled: v.string(),
+        enabled2: v.string(),
+        staged: v.string(),
+      })
+        .searchIndex("by_enabled", { searchField: "enabled" })
+        .searchIndex("by_enabled2", { searchField: "enabled2", staged: false })
+        .searchIndex("by_staged", { searchField: "staged", staged: true }),
+    });
+    type SearchIndexes = DataModelFromSchemaDefinition<
+      typeof schema
+    >["table"]["searchIndexes"];
+    type ExpectedSearchIndexes = {
+      by_enabled: {
+        searchField: "enabled";
+        filterFields: never;
+      };
+      by_enabled2: {
+        searchField: "enabled2";
+        filterFields: never;
+      };
+    };
+    assert<Equals<SearchIndexes, ExpectedSearchIndexes>>();
+  });
+
+  test("defineSchema creates staged vector indexes", () => {
+    const schema = defineSchema({
+      table: defineTable({
+        enabled: v.string(),
+        enabled2: v.string(),
+        staged: v.string(),
+      })
+        .vectorIndex("by_enabled", {
+          vectorField: "enabled",
+          dimensions: 1536,
+        })
+        .vectorIndex("by_enabled2", {
+          vectorField: "enabled2",
+          dimensions: 1536,
+          staged: false,
+        })
+        .vectorIndex("by_staged", {
+          vectorField: "staged",
+          dimensions: 1536,
+          staged: true,
+        }),
+    });
+    type VectorIndexes = DataModelFromSchemaDefinition<
+      typeof schema
+    >["table"]["vectorIndexes"];
+    type ExpectedVectorIndexes = {
+      by_enabled: {
+        vectorField: "enabled";
+        dimensions: number;
+        filterFields: never;
+      };
+      by_enabled2: {
+        vectorField: "enabled2";
+        dimensions: number;
+        filterFields: never;
+      };
+    };
+    assert<Equals<VectorIndexes, ExpectedVectorIndexes>>();
+  });
 });
 
 test("defineSchema generates search index types", () => {

npm-packages/convex/src/server/schema.ts

@@ -147,6 +147,26 @@ export type SearchIndex = {
   searchField: string;
   filterFields: string[];
 };
+
+/**
+ * Options for defining an index.
+ *
+ * @public
+ */
+export interface IndexOptions {
+  /**
+   * Whether the index should be staged.
+   *
+   * For large tables, index backfill can be slow. Staging an index allows you
+   * to push the schema and enable the index later.
+   *
+   * If `staged` is `true`, the index will be staged and will not be enabled
+   * until the staged flag is removed. Staged indexes do not block push
+   * completion. Staged indexes cannot be used in queries.
+   */
+  staged?: boolean;
+}
+
 /**
  * The definition of a table within a schema.
  *
@@ -210,10 +230,9 @@ export class TableDefinition<
   >(
     name: IndexName,
     fields: [FirstFieldPath, ...RestFieldPaths],
+    options?: IndexOptions & { staged: false },
   ): TableDefinition<
     DocumentType,
-    // Update `Indexes` to include the new index and use `Expand` to make the
-    // types look pretty in editors.
     Expand<
       Indexes &
         Record<
@@ -223,28 +242,49 @@ export class TableDefinition<
     >,
     SearchIndexes,
     VectorIndexes
-  > {
-    this.indexes.push({ indexDescriptor: name, fields });
-    return this;
-  }
+  >;
 
   /**
+   * Define a staged index on this table.
    *
-   * @internal
-   * @param name - The name of the staged index.
+   * For large tables, index backfill can be slow. Staging an index allows you
+   * to push the schema and enable the index later.
+   *
+   * If `staged` is `true`, the index will be staged and will not be enabled
+   * until the staged flag is removed. Staged indexes do not block push
+   * completion. Staged indexes cannot be used in queries.
+   *
+   * To learn about indexes, see [Defining Indexes](https://docs.convex.dev/using/indexes).
+   *
+   * @param name - The name of the index.
    * @param fields - The fields to index, in order. Must specify at least one
    * field.
-   * @returns A {@link TableDefinition} with this staged index included.
+   * @returns A {@link TableDefinition} with this index included.
    */
-  stagedIndex<
+  index<
+    IndexName extends string,
+    FirstFieldPath extends ExtractFieldPaths<DocumentType>,
+    RestFieldPaths extends ExtractFieldPaths<DocumentType>[],
+  >(
+    name: IndexName,
+    fields: [FirstFieldPath, ...RestFieldPaths],
+    options: IndexOptions & { staged: true },
+  ): TableDefinition<DocumentType, Indexes, SearchIndexes, VectorIndexes>;
+
+  index<
     IndexName extends string,
     FirstFieldPath extends ExtractFieldPaths<DocumentType>,
     RestFieldPaths extends ExtractFieldPaths<DocumentType>[],
   >(
     name: IndexName,
     fields: [FirstFieldPath, ...RestFieldPaths],
-  ): TableDefinition<DocumentType, Indexes, SearchIndexes, VectorIndexes> {
-    this.stagedDbIndexes.push({ indexDescriptor: name, fields });
+    options?: IndexOptions,
+  ) {
+    if (options?.staged) {
+      this.stagedDbIndexes.push({ indexDescriptor: name, fields });
+    } else {
+      this.indexes.push({ indexDescriptor: name, fields });
+    }
     return this;
   }
 
@@ -263,7 +303,10 @@ export class TableDefinition<
     FilterFields extends ExtractFieldPaths<DocumentType> = never,
   >(
     name: IndexName,
-    indexConfig: Expand<SearchIndexConfig<SearchField, FilterFields>>,
+    indexConfig: Expand<
+      SearchIndexConfig<SearchField, FilterFields> &
+        IndexOptions & { staged?: false }
+    >,
   ): TableDefinition<
     DocumentType,
     Indexes,
@@ -280,38 +323,59 @@ export class TableDefinition<
         >
     >,
     VectorIndexes
-  > {
-    this.searchIndexes.push({
-      indexDescriptor: name,
-      searchField: indexConfig.searchField,
-      filterFields: indexConfig.filterFields || [],
-    });
-    return this;
-  }
+  >;
 
   /**
-   * Stage a search index on this table.
+   * Define a staged search index on this table.
    *
-   * To learn about text search indexes, see [Search](https://docs.convex.dev/text-search).
+   * For large tables, index backfill can be slow. Staging an index allows you
+   * to push the schema and enable the index later.
+   *
+   * If `staged` is `true`, the index will be staged and will not be enabled
+   * until the staged flag is removed. Staged indexes do not block push
+   * completion. Staged indexes cannot be used in queries.
+   *
+   * To learn about search indexes, see [Search](https://docs.convex.dev/text-search).
    *
-   * @internal
    * @param name - The name of the index.
-   * @param indexConfig - The text index configuration object.
-   * @returns A {@link TableDefinition} with this text index included.
+   * @param indexConfig - The search index configuration object.
+   * @returns A {@link TableDefinition} with this search index included.
    */
-  stagedSearchIndex<
+  searchIndex<
     IndexName extends string,
     SearchField extends ExtractFieldPaths<DocumentType>,
     FilterFields extends ExtractFieldPaths<DocumentType> = never,
   >(
     name: IndexName,
-    indexConfig: Expand<SearchIndexConfig<SearchField, FilterFields>>,
-  ): TableDefinition<DocumentType, Indexes, SearchIndexes, VectorIndexes> {
-    this.stagedSearchIndexes.push({
-      indexDescriptor: name,
-      searchField: indexConfig.searchField,
-      filterFields: indexConfig.filterFields || [],
-    });
+    indexConfig: Expand<
+      SearchIndexConfig<SearchField, FilterFields> &
+        IndexOptions & { staged: true }
+    >,
+  ): TableDefinition<DocumentType, Indexes, SearchIndexes, VectorIndexes>;
+
+  searchIndex<
+    IndexName extends string,
+    SearchField extends ExtractFieldPaths<DocumentType>,
+    FilterFields extends ExtractFieldPaths<DocumentType> = never,
+  >(
+    name: IndexName,
+    indexConfig: Expand<
+      SearchIndexConfig<SearchField, FilterFields> & IndexOptions
+    >,
+  ) {
+    if (indexConfig.staged) {
+      this.stagedSearchIndexes.push({
+        indexDescriptor: name,
+        searchField: indexConfig.searchField,
+        filterFields: indexConfig.filterFields || [],
+      });
+    } else {
+      this.searchIndexes.push({
+        indexDescriptor: name,
+        searchField: indexConfig.searchField,
+        filterFields: indexConfig.filterFields || [],
+      });
+    }
     return this;
   }
 
@@ -330,7 +394,10 @@ export class TableDefinition<
     FilterFields extends ExtractFieldPaths<DocumentType> = never,
   >(
     name: IndexName,
-    indexConfig: Expand<VectorIndexConfig<VectorField, FilterFields>>,
+    indexConfig: Expand<
+      VectorIndexConfig<VectorField, FilterFields> &
+        IndexOptions & { staged?: false }
+    >,
   ): TableDefinition<
     DocumentType,
     Indexes,
@@ -346,40 +413,61 @@ export class TableDefinition<
           }
         >
     >
-  > {
-    this.vectorIndexes.push({
-      indexDescriptor: name,
-      vectorField: indexConfig.vectorField,
-      dimensions: indexConfig.dimensions,
-      filterFields: indexConfig.filterFields || [],
-    });
-    return this;
-  }
+  >;
 
   /**
-   * Stage a vector index on this table.
+   * Define a staged vector index on this table.
+   *
+   * For large tables, index backfill can be slow. Staging an index allows you
+   * to push the schema and enable the index later.
+   *
+   * If `staged` is `true`, the index will be staged and will not be enabled
+   * until the staged flag is removed. Staged indexes do not block push
+   * completion. Staged indexes cannot be used in queries.
    *
    * To learn about vector indexes, see [Vector Search](https://docs.convex.dev/vector-search).
    *
-   * @internal
    * @param name - The name of the index.
    * @param indexConfig - The vector index configuration object.
    * @returns A {@link TableDefinition} with this vector index included.
    */
-  stagedVectorIndex<
+  vectorIndex<
     IndexName extends string,
     VectorField extends ExtractFieldPaths<DocumentType>,
     FilterFields extends ExtractFieldPaths<DocumentType> = never,
   >(
     name: IndexName,
-    indexConfig: Expand<VectorIndexConfig<VectorField, FilterFields>>,
-  ): TableDefinition<DocumentType, Indexes, SearchIndexes, VectorIndexes> {
-    this.stagedVectorIndexes.push({
-      indexDescriptor: name,
-      vectorField: indexConfig.vectorField,
-      dimensions: indexConfig.dimensions,
-      filterFields: indexConfig.filterFields || [],
-    });
+    indexConfig: Expand<
+      VectorIndexConfig<VectorField, FilterFields> &
+        IndexOptions & { staged: true }
+    >,
+  ): TableDefinition<DocumentType, Indexes, SearchIndexes, VectorIndexes>;
+
+  vectorIndex<
+    IndexName extends string,
+    VectorField extends ExtractFieldPaths<DocumentType>,
+    FilterFields extends ExtractFieldPaths<DocumentType> = never,
+  >(
+    name: IndexName,
+    indexConfig: Expand<
+      VectorIndexConfig<VectorField, FilterFields> & IndexOptions
+    >,
+  ) {
+    if (indexConfig.staged) {
+      this.stagedVectorIndexes.push({
+        indexDescriptor: name,
+        vectorField: indexConfig.vectorField,
+        dimensions: indexConfig.dimensions,
+        filterFields: indexConfig.filterFields || [],
+      });
+    } else {
+      this.vectorIndexes.push({
+        indexDescriptor: name,
+        vectorField: indexConfig.vectorField,
+        dimensions: indexConfig.dimensions,
+        filterFields: indexConfig.filterFields || [],
+      });
+    }
     return this;
   }