fixed a document/chunk limit

Author: RobThePCGuy

dist/vectordb/index.d.ts

@@ -149,6 +149,30 @@ export declare class VectorStore {
      * Record FTS success (resets circuit breaker)
      */
     private recordFtsSuccess;
+    /**
+     * Extract unsupported custom metadata field from LanceDB schema mismatch errors.
+     *
+     * Returns:
+     * - specific key (e.g., "character") for metadata.custom.character mismatch
+     * - CUSTOM_METADATA_ALL_FIELDS when metadata.custom itself is unsupported
+     * - null when error is unrelated
+     */
+    private extractUnsupportedCustomMetadataField;
+    /**
+     * Remove unsupported custom metadata field from chunks for schema compatibility.
+     *
+     * @param chunks - Source chunks
+     * @param field - Unsupported field name, or CUSTOM_METADATA_ALL_FIELDS to drop all custom metadata
+     * @returns Sanitized chunks and whether any changes were applied
+     */
+    private stripUnsupportedCustomMetadata;
+    /**
+     * Add chunks to existing table with automatic fallback for custom metadata schema mismatches.
+     *
+     * LanceDB infers struct fields from early inserts, so later custom metadata keys may fail.
+     * This method retries by stripping only unsupported custom fields when needed.
+     */
+    private addChunksWithSchemaFallback;
     /**
      * Initialize LanceDB and create table
      */

dist/vectordb/index.d.ts.map

@@ -56,6 +56,67 @@ describe('VectorStore', () => {
     return vector.map((x) => x / norm)
   }
 
+  describe('Custom metadata schema compatibility', () => {
+    it('should keep ingesting when later chunks introduce new custom metadata keys', async () => {
+      const metadataSchemaDbPath = makeTestDbPath('test-vectordb-custom-metadata-schema')
+      if (fs.existsSync(metadataSchemaDbPath)) {
+        fs.rmSync(metadataSchemaDbPath, { recursive: true })
+      }
+
+      try {
+        const store = new VectorStore({
+          dbPath: metadataSchemaDbPath,
+          tableName: 'chunks',
+        })
+        await store.initialize()
+
+        const firstChunk = createTestChunk(
+          'Timeline chunk',
+          '/test/timeline.md',
+          0,
+          createNormalizedVector(1)
+        )
+        firstChunk.metadata.custom = {
+          domain: 'series-canon',
+          type: 'timeline',
+        }
+        await store.insertChunks([firstChunk])
+
+        const secondChunk = createTestChunk(
+          'Character arc chunk',
+          '/test/character.md',
+          0,
+          createNormalizedVector(2)
+        )
+        secondChunk.metadata.custom = {
+          domain: 'series-canon',
+          type: 'character-arc',
+          character: 'ellie',
+        }
+
+        // Regression check:
+        // Previously this failed with:
+        // "Found field not in schema: metadata.custom.character at row 0"
+        await expect(store.insertChunks([secondChunk])).resolves.toBeUndefined()
+
+        const files = await store.listFiles()
+        expect(files.map((f) => f.filePath).sort()).toEqual(
+          ['/test/character.md', '/test/timeline.md'].sort()
+        )
+
+        const results = await store.search(createNormalizedVector(2), 'character arc', 10)
+        const characterResult = results.find((r) => r.filePath === '/test/character.md')
+        expect(characterResult).toBeDefined()
+        expect(characterResult?.metadata.custom?.['domain']).toBe('series-canon')
+        expect(characterResult?.metadata.custom?.['type']).toBe('character-arc')
+      } finally {
+        if (fs.existsSync(metadataSchemaDbPath)) {
+          fs.rmSync(metadataSchemaDbPath, { recursive: true })
+        }
+      }
+    })
+  })
+
   describe('FTS Index Creation and Migration', () => {
     it('should not write VectorStore operational logs to stdout', async () => {
       const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})