feat: optimize vector table size

Author: NaccOll

src/services/code-index/config-manager.ts

@@ -13,7 +13,7 @@ export class CodeIndexConfigManager {
 	private codebaseIndexEnabled: boolean = true
 	private embedderProvider: EmbedderProvider = "openai"
 	private vectorStoreProvider: "local" | "qdrant" = "qdrant"
-	private localVectorStoreDirectoryPlaceholder?: string
+	private localVectorStoreDirectory?: string
 	private modelId?: string
 	private modelDimension?: number
 	private openAiOptions?: ApiHandlerOptions
@@ -83,7 +83,7 @@ export class CodeIndexConfigManager {
 		// Update instance variables with configuration
 		this.codebaseIndexEnabled = codebaseIndexEnabled ?? true
 		this.vectorStoreProvider = codebaseIndexVectorStoreProvider ?? "qdrant"
-		this.localVectorStoreDirectoryPlaceholder = codebaseIndexLocalVectorStoreDirectory
+		this.localVectorStoreDirectory = codebaseIndexLocalVectorStoreDirectory
 		this.qdrantUrl = codebaseIndexQdrantUrl
 		this.qdrantApiKey = qdrantApiKey ?? ""
 		this.searchMinScore = codebaseIndexSearchMinScore
@@ -165,7 +165,7 @@ export class CodeIndexConfigManager {
 			configured: this.isConfigured(),
 			embedderProvider: this.embedderProvider,
 			vectorStoreProvider: this.vectorStoreProvider,
-			localVectorStoreDirectoryPlaceholder: this.localVectorStoreDirectoryPlaceholder,
+			localVectorStoreDirectory: this.localVectorStoreDirectory,
 			modelId: this.modelId,
 			modelDimension: this.modelDimension,
 			openAiKey: this.openAiOptions?.openAiNativeApiKey ?? "",
@@ -272,7 +272,7 @@ export class CodeIndexConfigManager {
 		const prevQdrantUrl = prev?.qdrantUrl ?? ""
 		const prevQdrantApiKey = prev?.qdrantApiKey ?? ""
 		const prevVectorStoreProvider = prev?.vectorStoreProvider ?? "qdrant"
-		const prevLocalDbPath = prev?.localVectorStoreDirectoryPlaceholder ?? ""
+		const prevLocalDbPath = prev?.localVectorStoreDirectory ?? ""
 
 		// 1. Transition from disabled/unconfigured to enabled/configured
 		if ((!prevEnabled || !prevConfigured) && this.codebaseIndexEnabled && nowConfigured) {
@@ -306,10 +306,7 @@ export class CodeIndexConfigManager {
 		}
 
 		// Local DB path change (only affects local vector store)
-		if (
-			this.vectorStoreProvider === "local" &&
-			prevLocalDbPath !== (this.localVectorStoreDirectoryPlaceholder ?? "")
-		) {
+		if (this.vectorStoreProvider === "local" && prevLocalDbPath !== (this.localVectorStoreDirectory ?? "")) {
 			return true
 		}
 
@@ -323,7 +320,7 @@ export class CodeIndexConfigManager {
 		const currentMistralApiKey = this.mistralOptions?.apiKey ?? ""
 		const currentQdrantUrl = this.qdrantUrl ?? ""
 		const currentQdrantApiKey = this.qdrantApiKey ?? ""
-		const currentLocalDbPath = this.localVectorStoreDirectoryPlaceholder ?? ""
+		const currentLocalDbPath = this.localVectorStoreDirectory ?? ""
 
 		if (prevOpenAiKey !== currentOpenAiKey) {
 			return true
@@ -404,7 +401,7 @@ export class CodeIndexConfigManager {
 			isConfigured: this.isConfigured(),
 			embedderProvider: this.embedderProvider,
 			vectorStoreProvider: this.vectorStoreProvider ?? "qdrant",
-			localVectorStoreDirectoryPlaceholder: this.localVectorStoreDirectoryPlaceholder,
+			localVectorStoreDirectoryPlaceholder: this.localVectorStoreDirectory,
 			modelId: this.modelId,
 			modelDimension: this.modelDimension,
 			openAiOptions: this.openAiOptions,
@@ -502,6 +499,6 @@ export class CodeIndexConfigManager {
 	 * Gets the current local database path for vector storage
 	 */
 	public get currentLocalDbPath(): string | undefined {
-		return this.localVectorStoreDirectoryPlaceholder
+		return this.localVectorStoreDirectory
 	}
 }

src/services/code-index/interfaces/config.ts

@@ -30,7 +30,7 @@ export type PreviousConfigSnapshot = {
 	configured: boolean
 	embedderProvider: EmbedderProvider
 	vectorStoreProvider?: "local" | "qdrant"
-	localVectorStoreDirectoryPlaceholder?: string
+	localVectorStoreDirectory?: string
 	modelId?: string
 	modelDimension?: number // Generic dimension property
 	openAiKey?: string

src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts

@@ -1,4 +1,4 @@
-// npm vitest run src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts
+// npx vitest run services/code-index/vector-store/__tests__/local-vector-store.spec.ts
 
 /**
  * Comprehensive tests for LocalVectorStore.
@@ -9,38 +9,67 @@ import { describe, it, beforeEach, afterEach, expect, vi } from "vitest"
 import { LocalVectorStore } from "../local-vector-store"
 import { LanceDBManager } from "../../../lancedb-manager"
 import { Payload } from "../../interfaces"
+import * as path from "path"
 const fs = require("fs")
-
-const mockDb = {
-	tableNames: vi.fn(),
-	openTable: vi.fn(),
-	createTable: vi.fn(),
-	dropTable: vi.fn(),
-	connect: vi.fn(),
+const mockTable = {
+	delete: vi.fn().mockResolvedValue(undefined),
+	add: vi.fn().mockResolvedValue(undefined),
+	query: vi.fn().mockReturnThis(),
+	where: vi.fn().mockReturnThis(),
+	toArray: vi.fn().mockResolvedValue([]),
+	vectorSearch: vi.fn().mockReturnThis(),
+	limit: vi.fn().mockReturnThis(),
+	refineFactor: vi.fn().mockReturnThis(),
+	postfilter: vi.fn().mockReturnThis(),
+	openTable: vi.fn().mockResolvedValue(undefined),
+	search: vi.fn().mockReturnThis(),
+	name: "vector",
+	isOpen: true,
 	close: vi.fn(),
+	display: vi.fn(),
+	schema: {},
+	count: vi.fn(),
+	get: vi.fn(),
+	create: vi.fn(),
+	drop: vi.fn(),
+	insert: vi.fn(),
+	update: vi.fn(),
+	find: vi.fn(),
+	remove: vi.fn(),
+	createIndex: vi.fn(),
+	dropIndex: vi.fn(),
+	indexes: [],
+	columns: [],
+	primaryKey: "id",
+	metadata: {},
+	batch: vi.fn(),
+	distanceRange: vi.fn().mockReturnThis(),
 }
-const mockTable = {
-	query: vi.fn(),
-	add: vi.fn(),
-	delete: vi.fn(),
-	search: vi.fn(),
+const mockDb = {
+	openTable: vi.fn().mockResolvedValue(mockTable),
+	createTable: vi.fn().mockResolvedValue(mockTable),
+	dropTable: vi.fn().mockResolvedValue(undefined),
+	tableNames: vi.fn().mockResolvedValue(["vector", "metadata"]),
+	close: vi.fn().mockResolvedValue(undefined),
+	isOpen: true,
+	display: vi.fn(),
+	createEmptyTable: vi.fn(),
+	dropAllTables: vi.fn(),
 }
+
 const mockLanceDBModule = {
-	connect: vi.fn(() => Promise.resolve(mockDb)),
+	connect: vi.fn().mockResolvedValue(mockDb),
 }
 const mockLanceDBManager = {
 	ensureLanceDBAvailable: vi.fn(),
 	getNodeModulesPath: vi.fn(() => "mock_node_modules"),
 }
 
 vi.mock("@lancedb/lancedb", () => mockLanceDBModule)
-vi.mock("../../lancedb-manager", () => ({
-	LanceDBManager: vi.fn(() => mockLanceDBManager),
-}))
 
-const workspacePath = "/mock/workspace"
-const vectorSize = 3
-const dbDirectory = "/mock/db"
+const workspacePath = path.join("mock", "workspace")
+const vectorSize = 768
+const dbDirectory = path.join("mock", "db")
 let store: LocalVectorStore
 
 describe("LocalVectorStore", () => {

src/services/code-index/vector-store/local-vector-store.ts

@@ -213,9 +213,11 @@ export class LocalVectorStore implements IVectorStore {
 				await this._dropTableIfExists(db, this.metadataTableName)
 				await this._createVectorTable(db)
 				await this._createMetadataTable(db)
+				this.optimizeTable()
+
 				return true
 			}
-
+			this.optimizeTable()
 			return false
 		} catch (error) {
 			console.error(`[LocalVectorStore] Failed to initialize:`, error)
@@ -395,6 +397,9 @@ export class LocalVectorStore implements IVectorStore {
 			} catch (metadataError) {
 				console.warn("Failed to clear metadata table:", metadataError)
 			}
+
+			// Run optimization to clean up disk space after clearing
+			await this.optimizeTable()
 		} catch (error) {
 			console.error("Failed to clear collection:", error)
 			throw error
@@ -420,4 +425,22 @@ export class LocalVectorStore implements IVectorStore {
 			this.db = null
 		}
 	}
+
+	/**
+	 * Optimizes the table to reduce disk space usage and improve performance.
+	 * This method performs compaction, pruning of old versions, and index optimization.
+	 * Should be called periodically to prevent unbounded disk space growth.
+	 */
+	async optimizeTable(): Promise<void> {
+		try {
+			const table = await this.getTable()
+
+			await table.optimize({
+				cleanupOlderThan: new Date(),
+				deleteUnverified: false,
+			})
+		} catch (error) {
+			console.error("[LocalVectorStore] Failed to optimize table:", error)
+		}
+	}
 }