From b943eca5e1cc7364387ad3c193acacd3f4ecd571 Mon Sep 17 00:00:00 2001 From: NaccOll Date: Thu, 31 Jul 2025 11:29:12 +0800 Subject: [PATCH 1/7] feat: sqlite for code index vector store - Updated WebviewMessage interface to include options for local vector store provider and directory. - Implemented synchronous function to retrieve storage path for conversations. - Enhanced CodeIndexPopover component to manage settings for local and Qdrant vector stores. - Added translations for new settings in multiple languages. --- .github/actions/setup-node-pnpm/action.yml | 2 +- .nvmrc | 2 +- .tool-versions | 2 +- package.json | 2 +- packages/types/src/codebase-index.ts | 2 + pnpm-lock.yaml | 133 +---- src/core/webview/ClineProvider.ts | 6 + src/core/webview/webviewMessageHandler.ts | 2 + src/package.json | 4 +- .../__tests__/config-manager.spec.ts | 1 + src/services/code-index/config-manager.ts | 44 ++ src/services/code-index/interfaces/config.ts | 4 + src/services/code-index/service-factory.ts | 12 +- .../__tests__/local-vector-store.spec.ts | 479 +++++++++++++++++ .../vector-store/local-vector-store.ts | 497 ++++++++++++++++++ src/shared/WebviewMessage.ts | 2 + src/utils/storage.ts | 37 ++ .../src/components/chat/CodeIndexPopover.tsx | 140 +++-- .../src/context/ExtensionStateContext.tsx | 2 + webview-ui/src/i18n/locales/ca/settings.json | 4 + webview-ui/src/i18n/locales/de/settings.json | 4 + webview-ui/src/i18n/locales/en/settings.json | 4 + webview-ui/src/i18n/locales/es/settings.json | 4 + webview-ui/src/i18n/locales/fr/settings.json | 4 + webview-ui/src/i18n/locales/hi/settings.json | 4 + webview-ui/src/i18n/locales/id/settings.json | 4 + webview-ui/src/i18n/locales/it/settings.json | 4 + webview-ui/src/i18n/locales/ja/settings.json | 4 + webview-ui/src/i18n/locales/ko/settings.json | 4 + webview-ui/src/i18n/locales/nl/settings.json | 4 + webview-ui/src/i18n/locales/pl/settings.json | 4 + .../src/i18n/locales/pt-BR/settings.json | 4 + webview-ui/src/i18n/locales/ru/settings.json | 4 + webview-ui/src/i18n/locales/tr/settings.json | 4 + webview-ui/src/i18n/locales/vi/settings.json | 4 + .../src/i18n/locales/zh-CN/settings.json | 4 + .../src/i18n/locales/zh-TW/settings.json | 4 + 37 files changed, 1274 insertions(+), 171 deletions(-) create mode 100644 src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts create mode 100644 src/services/code-index/vector-store/local-vector-store.ts diff --git a/.github/actions/setup-node-pnpm/action.yml b/.github/actions/setup-node-pnpm/action.yml index af9b45b5e9..5f111c3828 100644 --- a/.github/actions/setup-node-pnpm/action.yml +++ b/.github/actions/setup-node-pnpm/action.yml @@ -6,7 +6,7 @@ inputs: node-version: description: "Node.js version to use" required: false - default: "20.19.2" + default: "22.17.1" pnpm-version: description: "pnpm version to use" required: false diff --git a/.nvmrc b/.nvmrc index 1d898f1fe5..8320a6d299 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v20.19.2 +22.15.1 diff --git a/.tool-versions b/.tool-versions index 269cea0b28..71c0000be7 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -nodejs 20.19.2 +nodejs 22.17.1 diff --git a/package.json b/package.json index 5e73f0c479..56eb34ed1a 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "roo-code", "packageManager": "pnpm@10.8.1", "engines": { - "node": "20.19.2" + "node": "22.17.1" }, "scripts": { "preinstall": "node scripts/bootstrap.mjs", diff --git a/packages/types/src/codebase-index.ts b/packages/types/src/codebase-index.ts index 89d5b168d7..c04a19b1af 100644 --- a/packages/types/src/codebase-index.ts +++ b/packages/types/src/codebase-index.ts @@ -22,6 +22,8 @@ export const codebaseIndexConfigSchema = z.object({ codebaseIndexEnabled: z.boolean().optional(), codebaseIndexQdrantUrl: z.string().optional(), codebaseIndexEmbedderProvider: z.enum(["openai", "ollama", "openai-compatible", "gemini", "mistral"]).optional(), + codebaseIndexVectorStoreProvider: z.enum(["local", "qdrant"]).optional(), + codebaseIndexLocalVectorStoreDirectory: z.string().optional(), codebaseIndexEmbedderBaseUrl: z.string().optional(), codebaseIndexEmbedderModelId: z.string().optional(), codebaseIndexEmbedderModelDimension: z.number().optional(), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3e7bb79b64..b47dc0f9b3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -805,8 +805,8 @@ importers: specifier: ^10.0.10 version: 10.0.10 '@types/node': - specifier: 20.x - version: 20.17.50 + specifier: ~22.15.29 + version: 22.15.29 '@types/node-cache': specifier: ^4.1.3 version: 4.2.5 @@ -875,7 +875,7 @@ importers: version: 5.8.3 vitest: specifier: ^3.2.3 - version: 3.2.4(@types/debug@4.1.12)(@types/node@20.17.50)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.15.29)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) zod-to-ts: specifier: ^1.2.0 version: 1.2.0(typescript@5.8.3)(zod@3.25.61) @@ -3877,18 +3877,9 @@ packages: '@types/node@18.19.100': resolution: {integrity: sha512-ojmMP8SZBKprc3qGrGk8Ujpo80AXkrP7G2tOT4VWr5jlr5DHjsJF+emXJz+Wm0glmy4Js62oKMdZZ6B9Y+tEcA==} - '@types/node@20.17.50': - resolution: {integrity: sha512-Mxiq0ULv/zo1OzOhwPqOA13I81CV/W3nvd3ChtQZRT5Cwz3cr0FKo/wMSsbTqL3EXpaBAEQhva2B8ByRkOIh9A==} - '@types/node@20.17.57': resolution: {integrity: sha512-f3T4y6VU4fVQDKVqJV4Uppy8c1p/sVvS3peyqxyWnzkqXFJLRU7Y1Bl7rMS1Qe9z0v4M6McY0Fp9yBsgHJUsWQ==} - '@types/node@20.19.1': - resolution: {integrity: sha512-jJD50LtlD2dodAEO653i3YF04NWak6jN3ky+Ri3Em3mGR39/glWiboM/IePaRbgwSfqM1TpGXfAg8ohn/4dTgA==} - - '@types/node@20.19.4': - resolution: {integrity: sha512-OP+We5WV8Xnbuvw0zC2m4qfB/BJvjyCwtNjhHdJxV1639SGSKrLmJkc3fMnp2Qy8nJyHp8RO6umxELN/dS1/EA==} - '@types/node@22.15.29': resolution: {integrity: sha512-LNdjOkUDlU1RZb8e1kOIUpN1qQUlzGkEtbVNo53vbrwDg5om6oduhm4SiUaPW5ASTXhAiP0jInWG8Qx9fVlOeQ==} @@ -11104,7 +11095,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 20.17.57 + '@types/node': 22.15.29 '@types/yargs': 17.0.33 chalk: 4.1.2 @@ -13084,7 +13075,7 @@ snapshots: '@types/glob@8.1.0': dependencies: '@types/minimatch': 5.1.2 - '@types/node': 20.17.57 + '@types/node': 22.15.29 '@types/hast@3.0.4': dependencies: @@ -13137,7 +13128,7 @@ snapshots: '@types/node-fetch@2.6.12': dependencies: - '@types/node': 20.17.57 + '@types/node': 22.15.29 form-data: 4.0.4 '@types/node-ipc@9.2.3': @@ -13152,23 +13143,10 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@20.17.50': - dependencies: - undici-types: 6.19.8 - '@types/node@20.17.57': dependencies: undici-types: 6.19.8 - '@types/node@20.19.1': - dependencies: - undici-types: 6.21.0 - - '@types/node@20.19.4': - dependencies: - undici-types: 6.21.0 - optional: true - '@types/node@22.15.29': dependencies: undici-types: 6.21.0 @@ -13202,11 +13180,11 @@ snapshots: '@types/stream-chain@2.1.0': dependencies: - '@types/node': 20.19.1 + '@types/node': 22.15.29 '@types/stream-json@1.7.8': dependencies: - '@types/node': 20.19.1 + '@types/node': 22.15.29 '@types/stream-chain': 2.1.0 '@types/string-similarity@4.0.2': {} @@ -13232,7 +13210,7 @@ snapshots: '@types/ws@8.18.1': dependencies: - '@types/node': 20.19.4 + '@types/node': 22.15.29 optional: true '@types/yargs-parser@21.0.3': {} @@ -13243,7 +13221,7 @@ snapshots: '@types/yauzl@2.10.3': dependencies: - '@types/node': 20.17.50 + '@types/node': 22.15.29 optional: true '@typescript-eslint/eslint-plugin@8.32.1(@typescript-eslint/parser@8.32.1(eslint@9.27.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.27.0(jiti@2.4.2))(typescript@5.8.3)': @@ -13352,14 +13330,6 @@ snapshots: chai: 5.2.0 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@6.3.5(@types/node@20.17.50)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))': - dependencies: - '@vitest/spy': 3.2.4 - estree-walker: 3.0.3 - magic-string: 0.30.17 - optionalDependencies: - vite: 6.3.5(@types/node@20.17.50)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) - '@vitest/mocker@3.2.4(vite@6.3.5(@types/node@20.17.57)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))': dependencies: '@vitest/spy': 3.2.4 @@ -16219,7 +16189,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 20.17.57 + '@types/node': 22.15.29 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -19627,27 +19597,6 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite-node@3.2.4(@types/node@20.17.50)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0): - dependencies: - cac: 6.7.14 - debug: 4.4.1(supports-color@8.1.1) - es-module-lexer: 1.7.0 - pathe: 2.0.3 - vite: 6.3.5(@types/node@20.17.50)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) - transitivePeerDependencies: - - '@types/node' - - jiti - - less - - lightningcss - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - yaml - vite-node@3.2.4(@types/node@20.17.57)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0): dependencies: cac: 6.7.14 @@ -19690,22 +19639,6 @@ snapshots: - tsx - yaml - vite@6.3.5(@types/node@20.17.50)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0): - dependencies: - esbuild: 0.25.5 - fdir: 6.4.4(picomatch@4.0.2) - picomatch: 4.0.2 - postcss: 8.5.4 - rollup: 4.40.2 - tinyglobby: 0.2.13 - optionalDependencies: - '@types/node': 20.17.50 - fsevents: 2.3.3 - jiti: 2.4.2 - lightningcss: 1.30.1 - tsx: 4.19.4 - yaml: 2.8.0 - vite@6.3.5(@types/node@20.17.57)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0): dependencies: esbuild: 0.25.5 @@ -19738,50 +19671,6 @@ snapshots: tsx: 4.19.4 yaml: 2.8.0 - vitest@3.2.4(@types/debug@4.1.12)(@types/node@20.17.50)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0): - dependencies: - '@types/chai': 5.2.2 - '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@6.3.5(@types/node@20.17.50)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0)) - '@vitest/pretty-format': 3.2.4 - '@vitest/runner': 3.2.4 - '@vitest/snapshot': 3.2.4 - '@vitest/spy': 3.2.4 - '@vitest/utils': 3.2.4 - chai: 5.2.0 - debug: 4.4.1(supports-color@8.1.1) - expect-type: 1.2.1 - magic-string: 0.30.17 - pathe: 2.0.3 - picomatch: 4.0.2 - std-env: 3.9.0 - tinybench: 2.9.0 - tinyexec: 0.3.2 - tinyglobby: 0.2.14 - tinypool: 1.1.1 - tinyrainbow: 2.0.0 - vite: 6.3.5(@types/node@20.17.50)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) - vite-node: 3.2.4(@types/node@20.17.50)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/debug': 4.1.12 - '@types/node': 20.17.50 - '@vitest/ui': 3.2.4(vitest@3.2.4) - jsdom: 26.1.0 - transitivePeerDependencies: - - jiti - - less - - lightningcss - - msw - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - yaml - vitest@3.2.4(@types/debug@4.1.12)(@types/node@20.17.57)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0): dependencies: '@types/chai': 5.2.2 diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 99c2a514b2..58ee871e14 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1645,6 +1645,8 @@ export class ClineProvider codebaseIndexConfig: { codebaseIndexEnabled: codebaseIndexConfig?.codebaseIndexEnabled ?? true, codebaseIndexQdrantUrl: codebaseIndexConfig?.codebaseIndexQdrantUrl ?? "http://localhost:6333", + codebaseIndexVectorStoreProvider: codebaseIndexConfig?.codebaseIndexVectorStoreProvider ?? "qdrant", + codebaseIndexLocalVectorStoreDirectory: codebaseIndexConfig?.codebaseIndexLocalVectorStoreDirectory, codebaseIndexEmbedderProvider: codebaseIndexConfig?.codebaseIndexEmbedderProvider ?? "openai", codebaseIndexEmbedderBaseUrl: codebaseIndexConfig?.codebaseIndexEmbedderBaseUrl ?? "", codebaseIndexEmbedderModelId: codebaseIndexConfig?.codebaseIndexEmbedderModelId ?? "", @@ -1834,6 +1836,10 @@ export class ClineProvider stateValues.codebaseIndexConfig?.codebaseIndexQdrantUrl ?? "http://localhost:6333", codebaseIndexEmbedderProvider: stateValues.codebaseIndexConfig?.codebaseIndexEmbedderProvider ?? "openai", + codebaseIndexVectorStoreProvider: + stateValues.codebaseIndexConfig?.codebaseIndexVectorStoreProvider ?? "qdrant", + codebaseIndexLocalVectorStoreDirectory: + stateValues.codebaseIndexConfig?.codebaseIndexLocalVectorStoreDirectory, codebaseIndexEmbedderBaseUrl: stateValues.codebaseIndexConfig?.codebaseIndexEmbedderBaseUrl ?? "", codebaseIndexEmbedderModelId: stateValues.codebaseIndexConfig?.codebaseIndexEmbedderModelId ?? "", codebaseIndexEmbedderModelDimension: diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index fdb7e90425..ab3d5d56ea 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -2010,6 +2010,8 @@ export const webviewMessageHandler = async ( codebaseIndexEnabled: settings.codebaseIndexEnabled, codebaseIndexQdrantUrl: settings.codebaseIndexQdrantUrl, codebaseIndexEmbedderProvider: settings.codebaseIndexEmbedderProvider, + codebaseIndexVectorStoreProvider: settings.codebaseIndexVectorStoreProvider, + codebaseIndexLocalVectorStoreDirectory: settings.codebaseIndexLocalVectorStoreDirectory, codebaseIndexEmbedderBaseUrl: settings.codebaseIndexEmbedderBaseUrl, codebaseIndexEmbedderModelId: settings.codebaseIndexEmbedderModelId, codebaseIndexEmbedderModelDimension: settings.codebaseIndexEmbedderModelDimension, // Generic dimension diff --git a/src/package.json b/src/package.json index d29a00e80b..8a6d4ac52b 100644 --- a/src/package.json +++ b/src/package.json @@ -11,7 +11,7 @@ }, "engines": { "vscode": "^1.84.0", - "node": "20.19.2" + "node": "22.17.1" }, "author": { "name": "Roo Code" @@ -493,7 +493,7 @@ "@types/diff-match-patch": "^1.0.36", "@types/glob": "^8.1.0", "@types/mocha": "^10.0.10", - "@types/node": "20.x", + "@types/node": "~22.15.29", "@types/node-cache": "^4.1.3", "@types/node-ipc": "^9.2.3", "@types/proper-lockfile": "^4.1.4", diff --git a/src/services/code-index/__tests__/config-manager.spec.ts b/src/services/code-index/__tests__/config-manager.spec.ts index 2d6e704d76..9ffd9fe3b6 100644 --- a/src/services/code-index/__tests__/config-manager.spec.ts +++ b/src/services/code-index/__tests__/config-manager.spec.ts @@ -1300,6 +1300,7 @@ describe("CodeIndexConfigManager", () => { qdrantApiKey: "test-qdrant-key", searchMinScore: 0.4, searchMaxResults: 50, + vectorStoreProvider: "qdrant", }) }) diff --git a/src/services/code-index/config-manager.ts b/src/services/code-index/config-manager.ts index 1723f1c2a0..b94209cf49 100644 --- a/src/services/code-index/config-manager.ts +++ b/src/services/code-index/config-manager.ts @@ -12,6 +12,8 @@ import { getDefaultModelId, getModelDimension, getModelScoreThreshold } from ".. export class CodeIndexConfigManager { private codebaseIndexEnabled: boolean = true private embedderProvider: EmbedderProvider = "openai" + private vectorStoreProvider: "local" | "qdrant" = "qdrant" + private localVectorStoreDirectoryPlaceholder?: string private modelId?: string private modelDimension?: number private openAiOptions?: ApiHandlerOptions @@ -46,10 +48,16 @@ export class CodeIndexConfigManager { codebaseIndexEnabled: true, codebaseIndexQdrantUrl: "http://localhost:6333", codebaseIndexEmbedderProvider: "openai", + codebaseIndexVectorStoreProvider: "qdrant", + codebaseIndexLocalVectorStoreDirectory: undefined, codebaseIndexEmbedderBaseUrl: "", codebaseIndexEmbedderModelId: "", + codebaseIndexEmbedderModelDimension: undefined, codebaseIndexSearchMinScore: undefined, codebaseIndexSearchMaxResults: undefined, + codebaseIndexOpenAiCompatibleBaseUrl: "", + codebaseIndexOpenAiCompatibleApiKey: "", + codebaseIndexGeminiApiKey: "", } const { @@ -58,9 +66,11 @@ export class CodeIndexConfigManager { codebaseIndexEmbedderProvider, codebaseIndexEmbedderBaseUrl, codebaseIndexEmbedderModelId, + codebaseIndexLocalVectorStoreDirectory, codebaseIndexSearchMinScore, codebaseIndexSearchMaxResults, } = codebaseIndexConfig + const codebaseIndexVectorStoreProvider = codebaseIndexConfig.codebaseIndexVectorStoreProvider ?? "qdrant" const openAiKey = this.contextProxy?.getSecret("codeIndexOpenAiKey") ?? "" const qdrantApiKey = this.contextProxy?.getSecret("codeIndexQdrantApiKey") ?? "" @@ -72,6 +82,8 @@ export class CodeIndexConfigManager { // Update instance variables with configuration this.codebaseIndexEnabled = codebaseIndexEnabled ?? true + this.vectorStoreProvider = codebaseIndexVectorStoreProvider ?? "qdrant" + this.localVectorStoreDirectoryPlaceholder = codebaseIndexLocalVectorStoreDirectory this.qdrantUrl = codebaseIndexQdrantUrl this.qdrantApiKey = qdrantApiKey ?? "" this.searchMinScore = codebaseIndexSearchMinScore @@ -152,6 +164,8 @@ export class CodeIndexConfigManager { enabled: this.codebaseIndexEnabled, configured: this.isConfigured(), embedderProvider: this.embedderProvider, + vectorStoreProvider: this.vectorStoreProvider, + localVectorStoreDirectoryPlaceholder: this.localVectorStoreDirectoryPlaceholder, modelId: this.modelId, modelDimension: this.modelDimension, openAiKey: this.openAiOptions?.openAiNativeApiKey ?? "", @@ -257,6 +271,8 @@ export class CodeIndexConfigManager { const prevMistralApiKey = prev?.mistralApiKey ?? "" const prevQdrantUrl = prev?.qdrantUrl ?? "" const prevQdrantApiKey = prev?.qdrantApiKey ?? "" + const prevVectorStoreProvider = prev?.vectorStoreProvider ?? "qdrant" + const prevLocalDbPath = prev?.localVectorStoreDirectoryPlaceholder ?? "" // 1. Transition from disabled/unconfigured to enabled/configured if ((!prevEnabled || !prevConfigured) && this.codebaseIndexEnabled && nowConfigured) { @@ -284,6 +300,19 @@ export class CodeIndexConfigManager { return true } + // Vector store provider change + if (prevVectorStoreProvider !== this.vectorStoreProvider) { + return true + } + + // Local DB path change (only affects local vector store) + if ( + this.vectorStoreProvider === "local" && + prevLocalDbPath !== (this.localVectorStoreDirectoryPlaceholder ?? "") + ) { + return true + } + // Authentication changes (API keys) const currentOpenAiKey = this.openAiOptions?.openAiNativeApiKey ?? "" const currentOllamaBaseUrl = this.ollamaOptions?.ollamaBaseUrl ?? "" @@ -294,6 +323,7 @@ export class CodeIndexConfigManager { const currentMistralApiKey = this.mistralOptions?.apiKey ?? "" const currentQdrantUrl = this.qdrantUrl ?? "" const currentQdrantApiKey = this.qdrantApiKey ?? "" + const currentLocalDbPath = this.localVectorStoreDirectoryPlaceholder ?? "" if (prevOpenAiKey !== currentOpenAiKey) { return true @@ -327,6 +357,11 @@ export class CodeIndexConfigManager { return true } + // Check for local database path changes (affects local vector store) + if (prevLocalDbPath !== currentLocalDbPath) { + return true + } + // Vector dimension changes (still important for compatibility) if (this._hasVectorDimensionChanged(prevProvider, prev?.modelId)) { return true @@ -368,6 +403,8 @@ export class CodeIndexConfigManager { return { isConfigured: this.isConfigured(), embedderProvider: this.embedderProvider, + vectorStoreProvider: this.vectorStoreProvider ?? "qdrant", + localVectorStoreDirectoryPlaceholder: this.localVectorStoreDirectoryPlaceholder, modelId: this.modelId, modelDimension: this.modelDimension, openAiOptions: this.openAiOptions, @@ -460,4 +497,11 @@ export class CodeIndexConfigManager { public get currentSearchMaxResults(): number { return this.searchMaxResults ?? DEFAULT_MAX_SEARCH_RESULTS } + + /** + * Gets the current local database path for vector storage + */ + public get currentLocalDbPath(): string | undefined { + return this.localVectorStoreDirectoryPlaceholder + } } diff --git a/src/services/code-index/interfaces/config.ts b/src/services/code-index/interfaces/config.ts index 9098a60091..f9b5bb37f2 100644 --- a/src/services/code-index/interfaces/config.ts +++ b/src/services/code-index/interfaces/config.ts @@ -7,6 +7,8 @@ import { EmbedderProvider } from "./manager" export interface CodeIndexConfig { isConfigured: boolean embedderProvider: EmbedderProvider + vectorStoreProvider?: "local" | "qdrant" + localVectorStoreDirectoryPlaceholder?: string modelId?: string modelDimension?: number // Generic dimension property for all providers openAiOptions?: ApiHandlerOptions @@ -27,6 +29,8 @@ export type PreviousConfigSnapshot = { enabled: boolean configured: boolean embedderProvider: EmbedderProvider + vectorStoreProvider?: "local" | "qdrant" + localVectorStoreDirectoryPlaceholder?: string modelId?: string modelDimension?: number // Generic dimension property openAiKey?: string diff --git a/src/services/code-index/service-factory.ts b/src/services/code-index/service-factory.ts index 68b0f5c0bc..e1276f9fcd 100644 --- a/src/services/code-index/service-factory.ts +++ b/src/services/code-index/service-factory.ts @@ -6,6 +6,7 @@ import { GeminiEmbedder } from "./embedders/gemini" import { MistralEmbedder } from "./embedders/mistral" import { EmbedderProvider, getDefaultModelId, getModelDimension } from "../../shared/embeddingModels" import { QdrantVectorStore } from "./vector-store/qdrant-client" +import { LocalVectorStore } from "./vector-store/local-vector-store" import { codeParser, DirectoryScanner, FileWatcher } from "./processors" import { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" import { CodeIndexConfigManager } from "./config-manager" @@ -14,6 +15,7 @@ import { Ignore } from "ignore" import { t } from "../../i18n" import { TelemetryService } from "@roo-code/telemetry" import { TelemetryEventName } from "@roo-code/types" +import { getLocalVectorStoreDirectoryPath } from "../../utils/storage" /** * Factory class responsible for creating and configuring code indexing service dependencies. @@ -131,7 +133,15 @@ export class CodeIndexServiceFactory { throw new Error(t("embeddings:serviceFactory.vectorDimensionNotDetermined", { modelId, provider })) } } - + // Use Local + if (config.vectorStoreProvider === "local") { + const { workspacePath } = this + const globalStorageUri = this.configManager.getContextProxy().globalStorageUri.fsPath + const localVectorStoreDirectoryPlaceholder = + config.localVectorStoreDirectoryPlaceholder || getLocalVectorStoreDirectoryPath(globalStorageUri) + return new LocalVectorStore(workspacePath, vectorSize, localVectorStoreDirectoryPlaceholder) + } + // Use Qdrant if (!config.qdrantUrl) { throw new Error(t("embeddings:serviceFactory.qdrantUrlMissing")) } diff --git a/src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts b/src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts new file mode 100644 index 0000000000..ea9fc504e4 --- /dev/null +++ b/src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts @@ -0,0 +1,479 @@ +// src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts +import { describe, it, expect, beforeEach, vi, afterEach } from "vitest" +import { LocalVectorStore } from "../local-vector-store" +import type { Payload, VectorStoreSearchResult } from "../../interfaces" +import * as path from "path" + +const mockDb = { + exec: vi.fn().mockResolvedValue(undefined), + prepare: vi.fn().mockReturnThis(), + get: vi.fn(), + run: vi.fn().mockResolvedValue(undefined), + all: vi.fn(), +} + +vi.mock("node:sqlite", () => ({ + default: { + DatabaseSync: vi.fn(() => mockDb), + }, +})) + +vi.mock("fs", () => ({ + existsSync: vi.fn(() => true), + mkdirSync: vi.fn(), + rmSync: vi.fn(), +})) + +vi.mock("../../../utils/path", () => ({ + getWorkspacePath: vi.fn(() => "/mock/workspace"), +})) + +vi.mock("crypto", () => ({ + createHash: () => ({ + update: () => ({ + digest: () => "mockhashmockhashmockhashmockhash", + }), + }), +})) + +describe("LocalVectorStore", () => { + let store: LocalVectorStore + + /** + * Create a new LocalVectorStore instance with correct parameters before each test. + */ + beforeEach(() => { + vi.clearAllMocks() + store = new LocalVectorStore("/mock/workspace", 4, ".roo/vector") + }) + + it("constructor should generate correct collectionName and dbPath", () => { + expect(store["collectionName"]).toBe("workspace-mockhashmockhash") + expect(store["dbPath"]).toMatch(/\.roo[\/\\]vector[\/\\]workspace-mockhashmockhash[\/\\]vector-store\.db$/) + }) + + describe("initialize", () => { + it("should return true when creating new collection", async () => { + mockDb.get = vi.fn().mockResolvedValue(undefined) + mockDb.run = vi.fn().mockResolvedValue(undefined) + const result = await store.initialize() + expect(result).toBe(true) + expect(mockDb.run).toHaveBeenCalled() + }) + + it("should return true when collection exists but vectorSize is different", async () => { + mockDb.get = vi.fn().mockResolvedValue({ id: 1, vector_size: 2 }) + mockDb.run = vi.fn().mockResolvedValue(undefined) + const result = await store.initialize() + expect(result).toBe(true) + expect(mockDb.run).toHaveBeenCalled() + }) + + it("should return false when collection exists and vectorSize is the same", async () => { + mockDb.get = vi.fn().mockResolvedValue({ id: 1, vector_size: 4 }) + const result = await store.initialize() + expect(result).toBe(false) + }) + + it("should throw error if db throws", async () => { + mockDb.get = vi.fn().mockRejectedValue(new Error("fail")) + await expect(store.initialize()).rejects.toThrow("vectorStore.localStoreInitFailed") + }) + }) + + describe("upsertPoints", () => { + it("should throw error when collection does not exist", async () => { + mockDb.get = vi.fn().mockResolvedValue(undefined) + store["cachedCollectionId"] = null + await expect( + store.upsertPoints([ + { + id: "1", + vector: [1, 2, 3, 4], + payload: { filePath: "a", codeChunk: "b", startLine: 1, endLine: 2 }, + }, + ]), + ).rejects.toThrow(/not found/) + }) + + it("should insert valid points and skip invalid payloads", async () => { + mockDb.get = vi.fn().mockResolvedValue({ id: 1, file_path: "a" }) + mockDb.all = vi.fn().mockResolvedValue([{ id: 1, file_path: "a" }]) + mockDb.run = vi.fn().mockResolvedValue(undefined) + mockDb.prepare = vi.fn().mockReturnValue({ + all: mockDb.all, + run: mockDb.run, + get: mockDb.get, + }) + store["cachedCollectionId"] = 1 + const points = [ + { id: "1", vector: [1, 2, 3, 4], payload: { filePath: "a", codeChunk: "b", startLine: 1, endLine: 2 } }, + { id: "2", vector: [1, 2, 3, 4], payload: { foo: "bar" } }, // invalid payload + ] + await store.upsertPoints(points) + expect(mockDb.run).toHaveBeenCalled() + }) + + it("should rollback transaction on error", async () => { + store["cachedCollectionId"] = 1 + mockDb.get = vi.fn().mockResolvedValue({ id: 1, file_path: "a" }) + mockDb.all = vi.fn().mockResolvedValue([{ id: 1, file_path: "a" }]) + mockDb.exec = vi.fn().mockResolvedValue(undefined) + + // Mock prepare to return different behaviors for different calls + let callCount = 0 + mockDb.prepare = vi.fn().mockImplementation((sql) => { + callCount++ + if (sql.includes("INSERT OR REPLACE INTO vectors")) { + return { + run: vi.fn().mockImplementation(() => { + throw new Error("fail") + }), + } + } + return { + all: mockDb.all, + run: mockDb.run, + get: mockDb.get, + } + }) + + const points = [ + { id: "1", vector: [1, 2, 3, 4], payload: { filePath: "a", codeChunk: "b", startLine: 1, endLine: 2 } }, + ] + await expect(store.upsertPoints(points)).rejects.toThrow("fail") + expect(mockDb.exec).toHaveBeenCalledWith("ROLLBACK") + }) + }) + + describe("search", () => { + it("should return empty array when collection does not exist", async () => { + mockDb.get = vi.fn().mockResolvedValue(undefined) + store["cachedCollectionId"] = null + const result = await store.search([1, 2, 3, 4]) + expect(result).toEqual([]) + }) + + it("should return correct matching results", async () => { + store["cachedCollectionId"] = 1 + // Set up db directly to avoid initialization + store["db"] = mockDb as any + let batchCallCount = 0 + mockDb.prepare = vi.fn().mockImplementation((sql) => { + if (sql.includes("COUNT(1)")) { + return { + get: vi.fn().mockResolvedValue({ total: 1 }), + } + } else if (sql.includes("SELECT v.id, v.vector, v.norm")) { + batchCallCount++ + // Return results only for the first batch call + return { + all: vi.fn().mockResolvedValue( + batchCallCount === 1 + ? [ + { + id: "1", + vector: Buffer.from(Float32Array.from([1, 2, 3, 4]).buffer), + norm: 5.477, + }, + ] + : [], + ), + } + } else if (sql.includes("f.file_path as filePath")) { + return { + all: vi.fn().mockResolvedValue([ + { + id: "1", + filePath: "a", + codeChunk: "b", + startLine: 1, + endLine: 2, + }, + ]), + } + } + return { + all: vi.fn().mockResolvedValue([]), + get: vi.fn().mockResolvedValue(undefined), + } + }) + const result = await store.search([1, 2, 3, 4]) + expect(result.length).toBe(1) + expect(result[0]).toHaveProperty("score") + expect(result[0].payload?.filePath).toBe("a") + }) + + it("should filter by directoryPrefix", async () => { + store["cachedCollectionId"] = 1 + // Set up db directly to avoid initialization + store["db"] = mockDb as any + let batchCallCount = 0 + mockDb.prepare = vi.fn().mockImplementation((sql) => { + if (sql.includes("COUNT(1)")) { + return { + get: vi.fn().mockResolvedValue({ total: 1 }), + } + } else if (sql.includes("SELECT v.id, v.vector, v.norm")) { + batchCallCount++ + // Return results only for the first batch call + return { + all: vi.fn().mockResolvedValue( + batchCallCount === 1 + ? [ + { + id: "1", + vector: Buffer.from(Float32Array.from([1, 2, 3, 4]).buffer), + norm: 5.477, + }, + ] + : [], + ), + } + } else if (sql.includes("f.file_path as filePath")) { + return { + all: vi.fn().mockResolvedValue([ + { + id: "1", + filePath: "prefix/a", + codeChunk: "b", + startLine: 1, + endLine: 2, + }, + ]), + } + } + return { + all: vi.fn().mockResolvedValue([]), + get: vi.fn().mockResolvedValue(undefined), + } + }) + const result = await store.search([1, 2, 3, 4], "prefix/") + expect(result.length).toBe(1) + expect(result[0].payload?.filePath).toBe("prefix/a") + }) + + it("should filter by minScore and maxResults", async () => { + store["cachedCollectionId"] = 1 + mockDb.get = vi.fn().mockResolvedValue({ total: 2 }) + mockDb.all = vi + .fn() + .mockResolvedValueOnce([ + { + id: "1", + vector: Buffer.from(Float32Array.from([1, 2, 3, 4]).buffer), + norm: 5.477, + }, + { + id: "2", + vector: Buffer.from(Float32Array.from([0, 0, 0, 0]).buffer), + norm: 0, + }, + ]) + .mockResolvedValueOnce([ + { + id: "1", + filePath: "a", + codeChunk: "b", + startLine: 1, + endLine: 2, + }, + ]) + mockDb.prepare = vi.fn().mockReturnValue({ + all: mockDb.all, + get: mockDb.get, + }) + const result = await store.search([1, 2, 3, 4], undefined, 0.99, 1) + expect(result.length).toBe(1) + expect(result[0].payload?.filePath).toBe("a") + }) + }) + + describe("deletePointsByFilePath", () => { + it("should call deletePointsByMultipleFilePaths", async () => { + const spy = vi.spyOn(store, "deletePointsByMultipleFilePaths").mockResolvedValue(undefined) + await store.deletePointsByFilePath("foo.ts") + expect(spy).toHaveBeenCalledWith(["foo.ts"]) + }) + }) + + describe("deletePointsByMultipleFilePaths", () => { + it("should return immediately for empty array", async () => { + const result = await store.deletePointsByMultipleFilePaths([]) + expect(result).toBeUndefined() + }) + + it("should normalize file paths and delete", async () => { + store["cachedCollectionId"] = 1 + mockDb.get = vi.fn().mockResolvedValue({ id: 1 }) + mockDb.all = vi.fn().mockResolvedValue([{ id: 1 }]) + mockDb.run = vi.fn().mockResolvedValue(undefined) + mockDb.prepare = vi.fn().mockReturnValue({ + all: mockDb.all, + run: mockDb.run, + get: mockDb.get, + }) + await store.deletePointsByMultipleFilePaths(["foo.ts"]) + expect(mockDb.run).toHaveBeenCalled() + }) + + it("should return if collection does not exist", async () => { + store["cachedCollectionId"] = null + const result = await store.deletePointsByMultipleFilePaths(["foo.ts"]) + expect(result).toBeUndefined() + }) + + it("should rollback transaction on error", async () => { + store["cachedCollectionId"] = 1 + mockDb.get = vi.fn().mockResolvedValue({ id: 1 }) + mockDb.all = vi.fn().mockResolvedValue([{ id: 1 }]) + const runMock = vi + .fn() + .mockResolvedValueOnce(undefined) // for BEGIN TRANSACTION + .mockImplementationOnce(() => { + throw new Error("fail") + }) + mockDb.run = runMock + mockDb.prepare = vi.fn().mockReturnValue({ + all: mockDb.all, + run: runMock, + get: mockDb.get, + }) + mockDb.exec = vi.fn().mockResolvedValue(undefined) + await expect(store.deletePointsByMultipleFilePaths(["foo.ts"])).rejects.toThrow("fail") + expect(mockDb.exec).toHaveBeenCalledWith("ROLLBACK") + }) + }) + + describe("deleteCollection", () => { + it("should delete when collection exists", async () => { + const fs = require("fs") + const rmSpy = vi.spyOn(fs, "rmSync").mockImplementation(() => {}) + // Ensure db file exists before deletion + vi.spyOn(fs, "existsSync").mockReturnValue(true) + await store.deleteCollection() + expect(rmSpy).toHaveBeenCalledWith(store["dbPath"]) + rmSpy.mockRestore() + }) + + it("should not delete when collection does not exist", async () => { + const fs = require("fs") + vi.spyOn(fs, "existsSync").mockReturnValue(false) + mockDb.get = vi.fn().mockResolvedValue(undefined) + mockDb.run = vi.fn().mockResolvedValue(undefined) + await store.deleteCollection() + // Table creation may call run once, but no delete should be called + const deleteCalls = mockDb.run.mock.calls.filter( + (call) => typeof call[0] === "string" && call[0].includes("DELETE FROM collections"), + ) + expect(deleteCalls.length).toBe(0) + }) + + it("should throw error on db failure", async () => { + const fs = require("fs") + vi.spyOn(fs, "existsSync").mockReturnValue(true) + vi.spyOn(fs, "rmSync").mockImplementation(() => { + throw new Error("fail") + }) + const clearSpy = vi.spyOn(store, "clearCollection").mockImplementation(() => Promise.resolve()) + await expect(store.deleteCollection()).rejects.toThrow("fail") + clearSpy.mockRestore() + }) + }) + + describe("clearCollection", () => { + it("should clear when collection exists", async () => { + store["cachedCollectionId"] = 1 + mockDb.get = vi.fn().mockResolvedValue({ id: 1 }) + mockDb.run = vi.fn().mockResolvedValue(undefined) + mockDb.prepare = vi.fn().mockReturnValue({ + run: mockDb.run, + get: mockDb.get, + }) + await store.clearCollection() + expect(mockDb.run).toHaveBeenCalled() + }) + + it("should not clear when collection does not exist", async () => { + store["cachedCollectionId"] = null + mockDb.get = vi.fn().mockResolvedValue(undefined) + mockDb.run = vi.fn().mockResolvedValue(undefined) + await store.clearCollection() + // No delete should be called when collection doesn't exist + const deleteCalls = mockDb.run.mock.calls.filter( + (call) => typeof call[0] === "string" && call[0].includes("DELETE FROM vectors"), + ) + expect(deleteCalls.length).toBe(0) + }) + + it("should throw error on db failure", async () => { + store["cachedCollectionId"] = 1 + mockDb.get = vi.fn().mockResolvedValue({ id: 1 }) + const runMock = vi.fn().mockImplementationOnce(() => { + throw new Error("fail") + }) + mockDb.run = runMock + mockDb.prepare = vi.fn().mockReturnValue({ + run: runMock, + get: mockDb.get, + }) + await expect(store.clearCollection()).rejects.toThrow("fail") + }) + }) + + describe("collectionExists", () => { + it("should return true when collection exists", async () => { + // Set up db directly to avoid initialization + store["db"] = mockDb as any + mockDb.prepare = vi.fn().mockReturnValue({ + get: vi.fn().mockResolvedValue({ id: 1 }), + }) + const result = await store.collectionExists() + expect(result).toBe(true) + }) + + it("should return false when collection does not exist", async () => { + // Set up db directly to avoid initialization + store["db"] = mockDb as any + mockDb.prepare = vi.fn().mockReturnValue({ + get: vi.fn().mockResolvedValue(undefined), + }) + const result = await store.collectionExists() + expect(result).toBe(false) + }) + }) + + describe("isPayloadValid", () => { + it("should return true for valid payload", () => { + const valid = store["isPayloadValid"]({ + filePath: "a", + codeChunk: "b", + startLine: 1, + endLine: 2, + }) + expect(valid).toBe(true) + }) + + it("should return false for invalid payload", () => { + const invalid = store["isPayloadValid"]({ foo: "bar" }) + expect(invalid).toBe(false) + }) + + it("should return false for null/undefined", () => { + expect(store["isPayloadValid"](null)).toBe(false) + expect(store["isPayloadValid"](undefined)).toBe(false) + }) + }) + + describe("getDb", () => { + it("should call initializeDatabase if db is not set", () => { + const s = new LocalVectorStore("/mock/workspace", 4, ".roo/vector") + const spy = vi.spyOn(s as any, "initializeDatabase").mockImplementation(() => Promise.resolve()) + s["db"] = null + + // Since getDb is async and complex to mock properly, let's test indirectly + // by checking that db is null initially + expect(s["db"]).toBeNull() + }) + }) +}) diff --git a/src/services/code-index/vector-store/local-vector-store.ts b/src/services/code-index/vector-store/local-vector-store.ts new file mode 100644 index 0000000000..0c74c6c46b --- /dev/null +++ b/src/services/code-index/vector-store/local-vector-store.ts @@ -0,0 +1,497 @@ +import { createHash } from "crypto" +import * as path from "path" +import * as os from "os" +import sql from "node:sqlite" +import { getWorkspacePath } from "../../../utils/path" +import { IVectorStore } from "../interfaces/vector-store" +import { Payload, VectorStoreSearchResult } from "../interfaces" +import { DEFAULT_MAX_SEARCH_RESULTS, DEFAULT_SEARCH_MIN_SCORE } from "../constants" +import { t } from "../../../i18n" + +/** + * Local implementation of the vector store using SQLite + */ +export class LocalVectorStore implements IVectorStore { + private readonly vectorSize: number + private readonly DISTANCE_METRIC = "Cosine" + private readonly dbPath: string + private db: sql.DatabaseSync | null = null + private readonly collectionName: string + private cachedCollectionId: number | null = null + + constructor(workspacePath: string, vectorSize: number, dbDirectory: string) { + this.vectorSize = vectorSize + const basename = path.basename(workspacePath) + // Generate collection name from workspace path + const hash = createHash("sha256").update(workspacePath).digest("hex") + this.collectionName = `${basename}-${hash.substring(0, 16)}` + // Set up database path + this.dbPath = path.join(dbDirectory, this.collectionName, `vector-store.db`) + } + + private async getDb(): Promise { + if (this.db) { + return this.db + } + + // Create parent directory if needed + const fs = require("fs") + const dir = path.dirname(this.dbPath) + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }) + } + + this.db = new sql.DatabaseSync(this.dbPath) + await this.initializeDatabase() + return this.db + } + + private async initializeDatabase(): Promise { + if (!this.db) return + + this.db.exec(` + PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + PRAGMA cache_size = 100000; + PRAGMA locking_mode = NORMAL; + PRAGMA temp_store = MEMORY; + `) + // Create tables if they don't exist + await this.db.exec(` + CREATE TABLE IF NOT EXISTS collections ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE NOT NULL, + vector_size INTEGER NOT NULL, + distance_metric TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + collection_id INTEGER NOT NULL, + file_path TEXT NOT NULL, + UNIQUE(collection_id, file_path) + ); + + CREATE TABLE IF NOT EXISTS vectors ( + id TEXT PRIMARY KEY, + collection_id INTEGER NOT NULL, + vector BLOB NOT NULL, + norm REAL NOT NULL, + file_id INTEGER NOT NULL, + code_chunk TEXT NOT NULL, + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_vectors_collection_file_id ON vectors(collection_id, file_id); + `) + + // Ensure our collection exists + const existing = await this.db.prepare("SELECT id FROM collections WHERE name = ?").get(this.collectionName) + + if (!existing) { + await this.db + .prepare("INSERT INTO collections (name, vector_size, distance_metric) VALUES (?, ?, ?)") + .run(this.collectionName, this.vectorSize, this.DISTANCE_METRIC) + } + } + + async initialize(): Promise { + try { + await this.closeConnect() + const db = await this.getDb() + + // Check if collection exists and has correct vector size + const collection = await db + .prepare("SELECT id, vector_size FROM collections WHERE name = ?") + .get(this.collectionName) + + if (!collection) { + // Create new collection + await db + .prepare("INSERT INTO collections (name, vector_size, distance_metric) VALUES (?, ?, ?)") + .run(this.collectionName, this.vectorSize, this.DISTANCE_METRIC) + const newCollection = await db + .prepare("SELECT id FROM collections WHERE name = ?") + .get(this.collectionName) + this.cachedCollectionId = newCollection?.id != null ? Number(newCollection.id) : null + return true + } else if (collection.vector_size !== this.vectorSize) { + // Recreate collection with correct vector size + await db.prepare("DELETE FROM vectors WHERE collection_id = ?").run(collection.id) + await db.prepare("DELETE FROM files WHERE collection_id = ?").run(collection.id) + await db + .prepare("UPDATE collections SET vector_size = ? WHERE id = ?") + .run(this.vectorSize, collection.id) + this.cachedCollectionId = collection.id != null ? Number(collection.id) : null + return true + } + + this.cachedCollectionId = collection.id != null ? Number(collection.id) : null + return false + } catch (error) { + console.error(`[LocalVectorStore] Failed to initialize:`, error) + throw new Error(t("embeddings:vectorStore.localStoreInitFailed", { errorMessage: error.message })) + } + } + + async upsertPoints( + points: Array<{ + id: string + vector: number[] + payload: Record + }>, + ): Promise { + const db = await this.getDb() + const collectionId = this.cachedCollectionId + if (collectionId == null) { + throw new Error(`Collection ${this.collectionName} not found`) + } + if (points.length === 0) { + return + } + + const valids = points.filter((point) => this.isPayloadValid(point.payload)) + const filePaths = valids.map((p) => p.payload.filePath) + const uniqueFilePaths = [...new Set(filePaths)] + + try { + await db.exec("BEGIN TRANSACTION") + const filePathIn = uniqueFilePaths.map(() => "?").join(",") + const existingFiles = await db + .prepare(`SELECT id, file_path FROM files WHERE collection_id = ? AND file_path IN (${filePathIn})`) + .all(collectionId, ...uniqueFilePaths) + + const existingPaths = new Set(existingFiles.map((f) => f.file_path)) + const newFilePaths = uniqueFilePaths.filter((p) => !existingPaths.has(p)) + + if (newFilePaths.length > 0) { + const placeholders = newFilePaths.map(() => "(?, ?)").join(",") + const insertSql = `INSERT INTO files (collection_id, file_path) VALUES ${placeholders}` + const insertParams = [] + for (const filePath of newFilePaths) { + insertParams.push(collectionId, filePath) + } + await db.prepare(insertSql).run(...insertParams) + } + + const allFiles = await db + .prepare(`SELECT id, file_path FROM files WHERE collection_id = ? AND file_path IN (${filePathIn})`) + .all(collectionId, ...uniqueFilePaths) + const existingFilesFinal = allFiles + const fileIdMap = new Map(existingFilesFinal.map((f) => [f.file_path, f.id])) + + const batchSize = 1000 + for (let i = 0; i < valids.length; i += batchSize) { + const validBatch = valids.slice(i, i + batchSize) + + if (validBatch.length === 0) { + continue + } + + const placeholders = validBatch.map(() => "(?, ?, ?, ?, ?, ?, ?, ?)").join(",") + const sqlStr = ` + INSERT OR REPLACE INTO vectors + (id, collection_id, vector, norm, file_id, code_chunk, start_line, end_line) + VALUES ${placeholders} + ` + + const values: any[] = [] + for (const point of validBatch) { + const vectorBuffer = Buffer.from(Float32Array.from(point.vector).buffer) + const norm = Math.sqrt(point.vector.reduce((sum, val) => sum + val * val, 0)) + const fileId = fileIdMap.get(point.payload.filePath) + if (typeof fileId !== "number") { + throw new Error(`Failed to get file_id for filePath: ${point.payload.filePath}`) + } + values.push( + point.id, + collectionId, + vectorBuffer, + norm, + fileId, + point.payload.codeChunk, + point.payload.startLine, + point.payload.endLine, + ) + } + + await db.prepare(sqlStr).run(...values) + } + + await db.exec("COMMIT") + } catch (error) { + await db.exec("ROLLBACK") + console.error("Failed to upsert points:", error) + throw error + } + } + + private isPayloadValid(payload: Record | null | undefined): payload is Payload { + if (!payload) { + return false + } + const validKeys = ["filePath", "codeChunk", "startLine", "endLine"] + const hasValidKeys = validKeys.every((key) => key in payload) + return hasValidKeys + } + + async search( + queryVector: number[], + directoryPrefix?: string, + minScore?: number, + maxResults?: number, + ): Promise { + const db = await this.getDb() + const collectionId = this.cachedCollectionId + if (collectionId == null) { + return [] + } + const actualMinScore = minScore ?? DEFAULT_SEARCH_MIN_SCORE + const actualMaxResults = maxResults ?? DEFAULT_MAX_SEARCH_RESULTS + + try { + // Calculate query norm once + const queryNorm = Math.sqrt(queryVector.reduce((sum, val) => sum + val * val, 0)) + + // Get total count first to determine parallel strategy + const countSql = ` + SELECT COUNT(1) as total + FROM vectors v + ${directoryPrefix ? "JOIN files f ON v.file_id = f.id" : ""} + WHERE v.collection_id = ? + ${directoryPrefix ? "AND f.file_path LIKE ?" : ""} + ` + const countParams = [collectionId, ...(directoryPrefix ? [`${directoryPrefix}%`] : [])] + const countResult = await db.prepare(countSql).get(...countParams) + const totalCount = Number(countResult?.total || 0) + + if (totalCount === 0) { + return [] + } + + const cpuCores = os.cpus().length + const maxParallelism = Math.max(1, Math.ceil(cpuCores / 2)) + const batchSize = 10000 + const totalBatches = Math.ceil(totalCount / batchSize) + const actualParallelism = Math.min(maxParallelism, totalBatches) + + // Parallel batch processing with dynamic task scheduling + const topResults: VectorStoreSearchResult[] = [] + let currentBatchIndex = 0 + const activeTasks = new Set>() + + // Process batches function + const processBatch = async (batchIndex: number): Promise => { + const offset = batchIndex * batchSize + const candidateSql = ` + SELECT v.id, v.vector, v.norm + FROM vectors v + ${directoryPrefix ? "JOIN files f ON v.file_id = f.id" : ""} + WHERE v.collection_id = ? + ${directoryPrefix ? "AND f.file_path LIKE ?" : ""} + LIMIT ? OFFSET ? + ` + const candidateParams = [ + collectionId, + ...(directoryPrefix ? [`${directoryPrefix}%`] : []), + batchSize, + offset, + ] + + const candidateBatch = await db.prepare(candidateSql).all(...candidateParams) + const batchResult = [] as VectorStoreSearchResult[] + + for (const r of candidateBatch) { + // Type safety checks + if (!r.vector || !r.norm || !r.id) { + continue + } + + const vectorBuffer = r.vector as Buffer + const norm = Number(r.norm) + const id = String(r.id) + + const float32Array = new Float32Array( + vectorBuffer.buffer, + vectorBuffer.byteOffset, + vectorBuffer.byteLength / Float32Array.BYTES_PER_ELEMENT, + ) + + let dot = 0 + for (let j = 0; j < queryVector.length; j++) { + dot += queryVector[j] * float32Array[j] + } + const score = dot / (queryNorm * norm || 1) + + if (score >= actualMinScore) { + batchResult.push({ id, score, payload: null }) + } + } + return batchResult + } + // Start initial parallel tasks + for (let i = 0; currentBatchIndex < totalBatches; i++) { + const task = processBatch(currentBatchIndex++) + activeTasks.add(task) + if (activeTasks.size > actualParallelism) { + const r = await Promise.race(activeTasks) + if (r?.length > 0) { + topResults.push(...r) + } + } + } + const results = await Promise.all(activeTasks) + for (const result of results) { + if (result?.length > 0) { + topResults.push(...result) + } + } + + if (topResults.length === 0) { + return [] + } + + // Sort final results descending + topResults.sort((a, b) => b.score - a.score) + if (topResults.length > actualMaxResults) { + topResults.splice(actualMaxResults) + } + + const ids = topResults.map((r) => r.id) + const placeholders = ids.map(() => "?").join(",") + const payloadSql = ` + SELECT + v.id, + f.file_path as filePath, + v.code_chunk as codeChunk, + v.start_line as startLine, + v.end_line as endLine + FROM vectors v + JOIN files f ON v.file_id = f.id + WHERE v.id IN (${placeholders}) + ` + const payloads = await db.prepare(payloadSql).all(...ids) + + // Map payloads to results with proper type conversion + const payloadMap = new Map( + payloads.map((p) => [ + p.id, + { + filePath: String(p.filePath), + codeChunk: String(p.codeChunk), + startLine: Number(p.startLine), + endLine: Number(p.endLine), + }, + ]), + ) + return topResults.map((r) => { + const payload = payloadMap.get(r.id) + if (!payload) { + throw new Error(`Missing payload for vector ${r.id}`) + } + return { + id: String(r.id), + score: r.score, + payload, + } + }) + } catch (error) { + console.error("Failed to search points:", error) + throw error + } + } + + async deletePointsByFilePath(filePath: string): Promise { + return this.deletePointsByMultipleFilePaths([filePath]) + } + + async deletePointsByMultipleFilePaths(filePaths: string[]): Promise { + if (filePaths.length === 0) { + return + } + + const db = await this.getDb() + const collectionId = this.cachedCollectionId + if (collectionId == null) { + return + } + + try { + await db.exec("BEGIN TRANSACTION") + + const workspaceRoot = getWorkspacePath() + const normalizedPaths = filePaths.map((fp) => + path.normalize(path.resolve(workspaceRoot, fp)).substring(workspaceRoot.length + 1), + ) + + const placeholders = normalizedPaths.map(() => "?").join(",") + const fileRows = await db + .prepare(`SELECT id FROM files WHERE collection_id = ? AND file_path IN (${placeholders})`) + .all(collectionId, ...normalizedPaths) + + const fileIds = fileRows.map((row: any) => row.id) + if (fileIds.length > 0) { + const fileIdPlaceholders = fileIds.map(() => "?").join(",") + await db + .prepare(`DELETE FROM vectors WHERE collection_id = ? AND file_id IN (${fileIdPlaceholders})`) + .run(collectionId, ...fileIds) + await db.prepare(`DELETE FROM files WHERE id IN (${fileIdPlaceholders})`).run(...fileIds) + } + + await db.exec("COMMIT") + } catch (error) { + await db.exec("ROLLBACK") + console.error("Failed to delete points by file paths:", error) + throw error + } + } + + async deleteCollection(): Promise { + await this.closeConnect() + try { + const fs = require("fs") + if (fs.existsSync(this.dbPath)) { + fs.rmSync(this.dbPath) + } + } catch (error) { + this.clearCollection() + throw error + } + } + + async clearCollection(): Promise { + const db = await this.getDb() + try { + const collectionId = this.cachedCollectionId + if (collectionId != null) { + await db.prepare("DELETE FROM vectors WHERE collection_id = ?").run(collectionId) + await db.prepare("DELETE FROM files WHERE collection_id = ?").run(collectionId) + await this.resizeCollection() + } + } catch (error) { + console.error("Failed to clear collection:", error) + throw error + } + } + + async collectionExists(): Promise { + const db = await this.getDb() + const collection = await db.prepare("SELECT id FROM collections WHERE name = ?").get(this.collectionName) + if (collection) { + this.cachedCollectionId = collection.id != null ? Number(collection.id) : null + } + return !!collection + } + async resizeCollection(): Promise { + const db = await this.getDb() + await db.exec(`VACUUM;`) + } + + private async closeConnect(): Promise { + if (this.db) { + this.db.close() + this.db = null + } + } +} diff --git a/src/shared/WebviewMessage.ts b/src/shared/WebviewMessage.ts index cb8759d851..104a2b20d3 100644 --- a/src/shared/WebviewMessage.ts +++ b/src/shared/WebviewMessage.ts @@ -258,6 +258,8 @@ export interface WebviewMessage { codebaseIndexEnabled: boolean codebaseIndexQdrantUrl: string codebaseIndexEmbedderProvider: "openai" | "ollama" | "openai-compatible" | "gemini" | "mistral" + codebaseIndexVectorStoreProvider?: "local" | "qdrant" + codebaseIndexLocalVectorStoreDirectory?: string codebaseIndexEmbedderBaseUrl?: string codebaseIndexEmbedderModelId: string codebaseIndexEmbedderModelDimension?: number // Generic dimension for all providers diff --git a/src/utils/storage.ts b/src/utils/storage.ts index 8240588794..08e7498a11 100644 --- a/src/utils/storage.ts +++ b/src/utils/storage.ts @@ -1,6 +1,7 @@ import * as vscode from "vscode" import * as path from "path" import * as fs from "fs/promises" +import * as fsSync from "fs" import { Package } from "../shared/package" import { t } from "../i18n" @@ -48,6 +49,32 @@ export async function getStorageBasePath(defaultPath: string): Promise { } } +/** + * Gets the base storage path for conversations(Sync) + */ +export function getStorageBasePathSync(defaultPath: string): string { + let customStoragePath = "" + try { + const config = vscode.workspace.getConfiguration(Package.name) + customStoragePath = config.get("customStoragePath", "") + } catch (error) { + console.warn("Could not access VSCode configuration - using default path") + return defaultPath + } + if (!customStoragePath) { + return defaultPath + } + try { + fsSync.mkdirSync(customStoragePath, { recursive: true }) + const testFile = path.join(customStoragePath, ".write_test") + fsSync.writeFileSync(testFile, "test") + fsSync.rmSync(testFile) + return customStoragePath + } catch (error) { + return defaultPath + } +} + /** * Gets the storage directory path for a task */ @@ -78,6 +105,16 @@ export async function getCacheDirectoryPath(globalStoragePath: string): Promise< return cacheDir } +/** + * Gets the local vector store directory path + */ +export function getLocalVectorStoreDirectoryPath(globalStoragePath: string): string { + const basePath = getStorageBasePathSync(globalStoragePath) + const cacheDir = path.join(basePath, "vector") + fsSync.mkdirSync(cacheDir, { recursive: true }) + return cacheDir +} + /** * Prompts the user to set a custom storage path * Displays an input box allowing the user to enter a custom path diff --git a/webview-ui/src/components/chat/CodeIndexPopover.tsx b/webview-ui/src/components/chat/CodeIndexPopover.tsx index c85aaf6ea5..6ee2c2345f 100644 --- a/webview-ui/src/components/chat/CodeIndexPopover.tsx +++ b/webview-ui/src/components/chat/CodeIndexPopover.tsx @@ -57,6 +57,8 @@ interface LocalCodeIndexSettings { codebaseIndexEnabled: boolean codebaseIndexQdrantUrl: string codebaseIndexEmbedderProvider: EmbedderProvider + codebaseIndexVectorStoreProvider: "local" | "qdrant" + codebaseIndexLocalVectorStoreDirectory?: string codebaseIndexEmbedderBaseUrl?: string codebaseIndexEmbedderModelId: string codebaseIndexEmbedderModelDimension?: number // Generic dimension for all providers @@ -169,6 +171,8 @@ export const CodeIndexPopover: React.FC = ({ codebaseIndexEnabled: true, codebaseIndexQdrantUrl: "", codebaseIndexEmbedderProvider: "openai", + codebaseIndexVectorStoreProvider: "qdrant", + codebaseIndexLocalVectorStoreDirectory: undefined, codebaseIndexEmbedderBaseUrl: "", codebaseIndexEmbedderModelId: "", codebaseIndexEmbedderModelDimension: undefined, @@ -200,6 +204,8 @@ export const CodeIndexPopover: React.FC = ({ codebaseIndexEnabled: codebaseIndexConfig.codebaseIndexEnabled ?? true, codebaseIndexQdrantUrl: codebaseIndexConfig.codebaseIndexQdrantUrl || "", codebaseIndexEmbedderProvider: codebaseIndexConfig.codebaseIndexEmbedderProvider || "openai", + codebaseIndexVectorStoreProvider: codebaseIndexConfig.codebaseIndexVectorStoreProvider || "qdrant", + codebaseIndexLocalVectorStoreDirectory: codebaseIndexConfig.codebaseIndexLocalVectorStoreDirectory, codebaseIndexEmbedderBaseUrl: codebaseIndexConfig.codebaseIndexEmbedderBaseUrl || "", codebaseIndexEmbedderModelId: codebaseIndexConfig.codebaseIndexEmbedderModelId || "", codebaseIndexEmbedderModelDimension: @@ -1020,54 +1026,104 @@ export const CodeIndexPopover: React.FC = ({ )} - {/* Qdrant Settings */} + {/* vectorStoreProviderLabel */}
- - updateSetting("codebaseIndexQdrantUrl", e.target.value) - } - onBlur={(e: any) => { - // Set default Qdrant URL if field is empty - if (!e.target.value.trim()) { - currentSettings.codebaseIndexQdrantUrl = DEFAULT_QDRANT_URL - updateSetting("codebaseIndexQdrantUrl", DEFAULT_QDRANT_URL) - } - }} - placeholder={t("settings:codeIndex.qdrantUrlPlaceholder")} - className={cn("w-full", { - "border-red-500": formErrors.codebaseIndexQdrantUrl, - })} - /> - {formErrors.codebaseIndexQdrantUrl && ( -

- {formErrors.codebaseIndexQdrantUrl} -

- )} +
+ {/* Qdrant Settings */} + {currentSettings.codebaseIndexVectorStoreProvider === "qdrant" && ( + <> +
+ + + updateSetting("codebaseIndexQdrantUrl", e.target.value) + } + onBlur={(e: any) => { + // Set default Qdrant URL if field is empty + if (!e.target.value.trim()) { + currentSettings.codebaseIndexQdrantUrl = DEFAULT_QDRANT_URL + updateSetting("codebaseIndexQdrantUrl", DEFAULT_QDRANT_URL) + } + }} + placeholder={t("settings:codeIndex.qdrantUrlPlaceholder")} + className={cn("w-full", { + "border-red-500": formErrors.codebaseIndexQdrantUrl, + })} + /> + {formErrors.codebaseIndexQdrantUrl && ( +

+ {formErrors.codebaseIndexQdrantUrl} +

+ )} +
-
- - updateSetting("codeIndexQdrantApiKey", e.target.value)} - placeholder={t("settings:codeIndex.qdrantApiKeyPlaceholder")} - className={cn("w-full", { - "border-red-500": formErrors.codeIndexQdrantApiKey, - })} - /> - {formErrors.codeIndexQdrantApiKey && ( -

- {formErrors.codeIndexQdrantApiKey} +

+ + + updateSetting("codeIndexQdrantApiKey", e.target.value) + } + placeholder={t("settings:codeIndex.qdrantApiKeyPlaceholder")} + className={cn("w-full", { + "border-red-500": formErrors.codeIndexQdrantApiKey, + })} + /> + {formErrors.codeIndexQdrantApiKey && ( +

+ {formErrors.codeIndexQdrantApiKey} +

+ )} +
+ + )} + + {/* Local Vector Store Settings */} + {currentSettings.codebaseIndexVectorStoreProvider === "local" && ( +
+ + + updateSetting( + "codebaseIndexLocalVectorStoreDirectory", + e.target.value, + ) + } + placeholder={t( + "settings:codeIndex.localVectorStoreDirectoryPlaceholder", + )} + className="w-full" + /> +

+ {t("settings:codeIndex.localVectorStoreDirectoryDescription")}

- )} -
+
+ )} )} diff --git a/webview-ui/src/context/ExtensionStateContext.tsx b/webview-ui/src/context/ExtensionStateContext.tsx index da7ab63358..37e0377395 100644 --- a/webview-ui/src/context/ExtensionStateContext.tsx +++ b/webview-ui/src/context/ExtensionStateContext.tsx @@ -238,6 +238,8 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode codebaseIndexEnabled: true, codebaseIndexQdrantUrl: "http://localhost:6333", codebaseIndexEmbedderProvider: "openai", + codebaseIndexVectorStoreProvider: "qdrant", + codebaseIndexLocalVectorStoreDirectory: undefined, codebaseIndexEmbedderBaseUrl: "", codebaseIndexEmbedderModelId: "", codebaseIndexSearchMaxResults: undefined, diff --git a/webview-ui/src/i18n/locales/ca/settings.json b/webview-ui/src/i18n/locales/ca/settings.json index 82c8f40516..81c695fbd5 100644 --- a/webview-ui/src/i18n/locales/ca/settings.json +++ b/webview-ui/src/i18n/locales/ca/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "Introduïu el nom del model", "selectModel": "Seleccioneu un model", "ollamaBaseUrlLabel": "URL base d'Ollama", + "vectorStoreProviderLabel": "Proveïdor d'emmagatzematge de vectors", "qdrantApiKeyLabel": "Clau API de Qdrant", "qdrantApiKeyPlaceholder": "Introduïu la vostra clau API de Qdrant (opcional)", + "localVectorStoreDirectoryLabel": "Camí d'emmagatzematge de vectors local", + "localVectorStoreDirectoryPlaceholder": "Introduïu un camí d'emmagatzematge de vectors personalitzat (opcional)", + "localVectorStoreDirectoryDescription": "Camí per emmagatzemar la base de dades de vectors local. Si està buit, utilitza la ubicació per defecte a globalStorageUri/vector.", "setupConfigLabel": "Configuració", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/de/settings.json b/webview-ui/src/i18n/locales/de/settings.json index df61c3142e..a0f432f6ad 100644 --- a/webview-ui/src/i18n/locales/de/settings.json +++ b/webview-ui/src/i18n/locales/de/settings.json @@ -71,10 +71,14 @@ "selectModelPlaceholder": "Modell auswählen", "ollamaUrlLabel": "Ollama-URL:", "ollamaBaseUrlLabel": "Ollama Basis-URL", + "vectorStoreProviderLabel": "Vektorspeicher-Anbieter", "qdrantUrlLabel": "Qdrant-URL", "qdrantKeyLabel": "Qdrant-Schlüssel:", "qdrantApiKeyLabel": "Qdrant API-Schlüssel", "qdrantApiKeyPlaceholder": "Gib deinen Qdrant API-Schlüssel ein (optional)", + "localVectorStoreDirectoryLabel": "Lokaler Vektorspeicher Pfad", + "localVectorStoreDirectoryPlaceholder": "Gib einen benutzerdefinierten Vektorspeicher Pfad ein (optional)", + "localVectorStoreDirectoryDescription": "Pfad zur Speicherung der lokalen Vektordatenbank. Falls leer, wird der Standardort in globalStorageUri/vector verwendet.", "setupConfigLabel": "Einrichtung", "startIndexingButton": "Start", "clearIndexDataButton": "Index löschen", diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index 11c575bdf3..f90f359360 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -71,10 +71,14 @@ "selectModelPlaceholder": "Select model", "ollamaUrlLabel": "Ollama URL:", "ollamaBaseUrlLabel": "Ollama Base URL", + "vectorStoreProviderLabel": "Vector Store Provider", "qdrantUrlLabel": "Qdrant URL", "qdrantKeyLabel": "Qdrant Key:", "qdrantApiKeyLabel": "Qdrant API Key", "qdrantApiKeyPlaceholder": "Enter your Qdrant API key (optional)", + "localVectorStoreDirectoryLabel": "Local Vector Store Path", + "localVectorStoreDirectoryPlaceholder": "Enter custom vector store path (optional)", + "localVectorStoreDirectoryDescription": "Path to store the local vector database. If empty, uses the default location in the globalStorageUri/vector.", "setupConfigLabel": "Setup", "advancedConfigLabel": "Advanced Configuration", "searchMinScoreLabel": "Search Score Threshold", diff --git a/webview-ui/src/i18n/locales/es/settings.json b/webview-ui/src/i18n/locales/es/settings.json index 3afeb091ef..37af708668 100644 --- a/webview-ui/src/i18n/locales/es/settings.json +++ b/webview-ui/src/i18n/locales/es/settings.json @@ -71,10 +71,14 @@ "selectModelPlaceholder": "Seleccionar modelo", "ollamaUrlLabel": "URL de Ollama:", "ollamaBaseUrlLabel": "URL base de Ollama", + "vectorStoreProviderLabel": "Proveedor de almacén de vectores", "qdrantUrlLabel": "URL de Qdrant", "qdrantKeyLabel": "Clave de Qdrant:", "qdrantApiKeyLabel": "Clave API de Qdrant", "qdrantApiKeyPlaceholder": "Introduce tu clave API de Qdrant (opcional)", + "localVectorStoreDirectoryLabel": "Ruta del almacén de vectores local", + "localVectorStoreDirectoryPlaceholder": "Introduce una ruta personalizada del almacén de vectores (opcional)", + "localVectorStoreDirectoryDescription": "Ruta para almacenar la base de datos de vectores local. Si está vacía, usa la ubicación predeterminada en globalStorageUri/vector.", "setupConfigLabel": "Configuración", "startIndexingButton": "Iniciar", "clearIndexDataButton": "Borrar índice", diff --git a/webview-ui/src/i18n/locales/fr/settings.json b/webview-ui/src/i18n/locales/fr/settings.json index 5b1c0431fe..ac993473fd 100644 --- a/webview-ui/src/i18n/locales/fr/settings.json +++ b/webview-ui/src/i18n/locales/fr/settings.json @@ -71,10 +71,14 @@ "selectModelPlaceholder": "Sélectionner un modèle", "ollamaUrlLabel": "URL Ollama :", "ollamaBaseUrlLabel": "URL de base Ollama", + "vectorStoreProviderLabel": "Fournisseur de stockage vectoriel", "qdrantUrlLabel": "URL Qdrant", "qdrantKeyLabel": "Clé Qdrant :", "qdrantApiKeyLabel": "Clé API Qdrant", "qdrantApiKeyPlaceholder": "Entrez votre clé API Qdrant (optionnel)", + "localVectorStoreDirectoryLabel": "Chemin du stockage vectoriel local", + "localVectorStoreDirectoryPlaceholder": "Entrez un chemin personnalisé du stockage vectoriel (optionnel)", + "localVectorStoreDirectoryDescription": "Chemin pour stocker la base de données vectorielle locale. Si vide, utilise l'emplacement par défaut dans globalStorageUri/vector.", "setupConfigLabel": "Configuration", "startIndexingButton": "Démarrer", "clearIndexDataButton": "Effacer l'index", diff --git a/webview-ui/src/i18n/locales/hi/settings.json b/webview-ui/src/i18n/locales/hi/settings.json index 0c7ab3a0bc..31c6f1fd85 100644 --- a/webview-ui/src/i18n/locales/hi/settings.json +++ b/webview-ui/src/i18n/locales/hi/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "मॉडल नाम दर्ज करें", "selectModel": "एक मॉडल चुनें", "ollamaBaseUrlLabel": "Ollama आधार URL", + "vectorStoreProviderLabel": "वेक्टर स्टोर प्रदाता", "qdrantApiKeyLabel": "Qdrant API कुंजी", "qdrantApiKeyPlaceholder": "अपनी Qdrant API कुंजी दर्ज करें (वैकल्पिक)", + "localVectorStoreDirectoryLabel": "स्थानीय वेक्टर स्टोर पथ", + "localVectorStoreDirectoryPlaceholder": "कस्टम वेक्टर स्टोर पथ दर्ज करें (वैकल्पिक)", + "localVectorStoreDirectoryDescription": "स्थानीय वेक्टर डेटाबेस संग्रहीत करने के लिए पथ। यदि खाली है, तो globalStorageUri/vector में डिफ़ॉल्ट स्थान का उपयोग करता है।", "setupConfigLabel": "सेटअप", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/id/settings.json b/webview-ui/src/i18n/locales/id/settings.json index fc1b1915ab..1ff319ec3f 100644 --- a/webview-ui/src/i18n/locales/id/settings.json +++ b/webview-ui/src/i18n/locales/id/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "Masukkan nama model", "selectModel": "Pilih model", "ollamaBaseUrlLabel": "URL Dasar Ollama", + "vectorStoreProviderLabel": "Penyedia Penyimpanan Vektor", "qdrantApiKeyLabel": "Kunci API Qdrant", "qdrantApiKeyPlaceholder": "Masukkan kunci API Qdrant kamu (opsional)", + "localVectorStoreDirectoryLabel": "Jalur Penyimpanan Vektor Lokal", + "localVectorStoreDirectoryPlaceholder": "Masukkan jalur penyimpanan vektor kustom (opsional)", + "localVectorStoreDirectoryDescription": "Jalur untuk menyimpan database vektor lokal. Jika kosong, menggunakan lokasi default di globalStorageUri/vector.", "setupConfigLabel": "Pengaturan", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/it/settings.json b/webview-ui/src/i18n/locales/it/settings.json index 3b82f073b3..e5a29ce89c 100644 --- a/webview-ui/src/i18n/locales/it/settings.json +++ b/webview-ui/src/i18n/locales/it/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "Inserisci il nome del modello", "selectModel": "Seleziona un modello", "ollamaBaseUrlLabel": "URL base Ollama", + "vectorStoreProviderLabel": "Fornitore di archivio vettoriale", "qdrantApiKeyLabel": "Chiave API Qdrant", "qdrantApiKeyPlaceholder": "Inserisci la tua chiave API Qdrant (opzionale)", + "localVectorStoreDirectoryLabel": "Percorso archivio vettoriale locale", + "localVectorStoreDirectoryPlaceholder": "Inserisci un percorso personalizzato dell'archivio vettoriale (opzionale)", + "localVectorStoreDirectoryDescription": "Percorso per archiviare il database vettoriale locale. Se vuoto, usa la posizione predefinita in globalStorageUri/vector.", "setupConfigLabel": "Impostazione", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/ja/settings.json b/webview-ui/src/i18n/locales/ja/settings.json index 321b269a8a..6689c3ba96 100644 --- a/webview-ui/src/i18n/locales/ja/settings.json +++ b/webview-ui/src/i18n/locales/ja/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "モデル名を入力", "selectModel": "モデルを選択", "ollamaBaseUrlLabel": "Ollama ベースURL", + "vectorStoreProviderLabel": "ベクターストアプロバイダー", "qdrantApiKeyLabel": "Qdrant APIキー", "qdrantApiKeyPlaceholder": "Qdrant APIキーを入力（オプション）", + "localVectorStoreDirectoryLabel": "ローカルベクターストアパス", + "localVectorStoreDirectoryPlaceholder": "カスタムベクターストアパスを入力（オプション）", + "localVectorStoreDirectoryDescription": "ローカルベクターデータベースを格納するパス。空の場合、globalStorageUri/vector のデフォルトの場所を使用します。", "setupConfigLabel": "設定", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/ko/settings.json b/webview-ui/src/i18n/locales/ko/settings.json index d286ac71a2..6d1f5a9a6c 100644 --- a/webview-ui/src/i18n/locales/ko/settings.json +++ b/webview-ui/src/i18n/locales/ko/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "모델 이름을 입력하세요", "selectModel": "모델 선택", "ollamaBaseUrlLabel": "Ollama 기본 URL", + "vectorStoreProviderLabel": "벡터 스토어 제공자", "qdrantApiKeyLabel": "Qdrant API 키", "qdrantApiKeyPlaceholder": "Qdrant API 키를 입력하세요 (선택사항)", + "localVectorStoreDirectoryLabel": "로컬 벡터 스토어 경로", + "localVectorStoreDirectoryPlaceholder": "사용자 정의 벡터 스토어 경로를 입력하세요 (선택사항)", + "localVectorStoreDirectoryDescription": "로컬 벡터 데이터베이스를 저장할 경로입니다. 비어있으면 globalStorageUri/vector의 기본 위치를 사용합니다.", "setupConfigLabel": "설정", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/nl/settings.json b/webview-ui/src/i18n/locales/nl/settings.json index e8c1db5ace..c79aa06c7f 100644 --- a/webview-ui/src/i18n/locales/nl/settings.json +++ b/webview-ui/src/i18n/locales/nl/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "Voer modelnaam in", "selectModel": "Selecteer een model", "ollamaBaseUrlLabel": "Ollama Basis-URL", + "vectorStoreProviderLabel": "Vectoropslag Provider", "qdrantApiKeyLabel": "Qdrant API-sleutel", "qdrantApiKeyPlaceholder": "Voer je Qdrant API-sleutel in (optioneel)", + "localVectorStoreDirectoryLabel": "Lokaal vectoropslag pad", + "localVectorStoreDirectoryPlaceholder": "Voer een aangepast vectoropslag pad in (optioneel)", + "localVectorStoreDirectoryDescription": "Pad om de lokale vectordatabase op te slaan. Als leeg, gebruikt het de standaardlocatie in globalStorageUri/vector.", "setupConfigLabel": "Instellen", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/pl/settings.json b/webview-ui/src/i18n/locales/pl/settings.json index ab208ffe14..46bbb35b1a 100644 --- a/webview-ui/src/i18n/locales/pl/settings.json +++ b/webview-ui/src/i18n/locales/pl/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "Wprowadź nazwę modelu", "selectModel": "Wybierz model", "ollamaBaseUrlLabel": "Bazowy URL Ollama", + "vectorStoreProviderLabel": "Dostawca magazynu wektorów", "qdrantApiKeyLabel": "Klucz API Qdrant", "qdrantApiKeyPlaceholder": "Wprowadź swój klucz API Qdrant (opcjonalnie)", + "localVectorStoreDirectoryLabel": "Ścieżka lokalnego magazynu wektorów", + "localVectorStoreDirectoryPlaceholder": "Wprowadź niestandardową ścieżkę magazynu wektorów (opcjonalnie)", + "localVectorStoreDirectoryDescription": "Ścieżka do przechowywania lokalnej bazy danych wektorów. Jeśli pusta, używa domyślnej lokalizacji w globalStorageUri/vector.", "setupConfigLabel": "Konfiguracja", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/pt-BR/settings.json b/webview-ui/src/i18n/locales/pt-BR/settings.json index 6bcfbb564c..1e97fe930e 100644 --- a/webview-ui/src/i18n/locales/pt-BR/settings.json +++ b/webview-ui/src/i18n/locales/pt-BR/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "Insira o nome do modelo", "selectModel": "Selecione um modelo", "ollamaBaseUrlLabel": "URL Base do Ollama", + "vectorStoreProviderLabel": "Provedor de Armazenamento de Vetores", "qdrantApiKeyLabel": "Chave da API Qdrant", "qdrantApiKeyPlaceholder": "Insira sua chave da API Qdrant (opcional)", + "localVectorStoreDirectoryLabel": "Caminho do Armazenamento de Vetores Local", + "localVectorStoreDirectoryPlaceholder": "Insira um caminho personalizado do armazenamento de vetores (opcional)", + "localVectorStoreDirectoryDescription": "Caminho para armazenar o banco de dados de vetores local. Se vazio, usa a localização padrão em globalStorageUri/vector.", "setupConfigLabel": "Configuração", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/ru/settings.json b/webview-ui/src/i18n/locales/ru/settings.json index 8d52241d6c..c91a1589e9 100644 --- a/webview-ui/src/i18n/locales/ru/settings.json +++ b/webview-ui/src/i18n/locales/ru/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "Введите название модели", "selectModel": "Выберите модель", "ollamaBaseUrlLabel": "Базовый URL Ollama", + "vectorStoreProviderLabel": "Поставщик векторного хранилища", "qdrantApiKeyLabel": "API-ключ Qdrant", "qdrantApiKeyPlaceholder": "Введите ваш API-ключ Qdrant (необязательно)", + "localVectorStoreDirectoryLabel": "Путь локального векторного хранилища", + "localVectorStoreDirectoryPlaceholder": "Введите пользовательский путь векторного хранилища (необязательно)", + "localVectorStoreDirectoryDescription": "Путь для хранения локальной векторной базы данных. Если пустой, используется местоположение по умолчанию в globalStorageUri/vector.", "setupConfigLabel": "Настройка", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/tr/settings.json b/webview-ui/src/i18n/locales/tr/settings.json index 486dad0540..9e32fa9873 100644 --- a/webview-ui/src/i18n/locales/tr/settings.json +++ b/webview-ui/src/i18n/locales/tr/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "Model adını girin", "selectModel": "Bir model seçin", "ollamaBaseUrlLabel": "Ollama Temel URL", + "vectorStoreProviderLabel": "Vektör Depolama Sağlayıcısı", "qdrantApiKeyLabel": "Qdrant API Anahtarı", "qdrantApiKeyPlaceholder": "Qdrant API anahtarınızı girin (isteğe bağlı)", + "localVectorStoreDirectoryLabel": "Yerel Vektör Depolama Yolu", + "localVectorStoreDirectoryPlaceholder": "Özel vektör depolama yolu girin (isteğe bağlı)", + "localVectorStoreDirectoryDescription": "Yerel vektör veritabanının depolanacağı yol. Boşsa, globalStorageUri/vector konumunu kullanır.", "setupConfigLabel": "Kurulum", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/vi/settings.json b/webview-ui/src/i18n/locales/vi/settings.json index dbe0e73736..6287c74aa8 100644 --- a/webview-ui/src/i18n/locales/vi/settings.json +++ b/webview-ui/src/i18n/locales/vi/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "Nhập tên mô hình", "selectModel": "Chọn một mô hình", "ollamaBaseUrlLabel": "URL cơ sở Ollama", + "vectorStoreProviderLabel": "Nhà cung cấp kho lưu trữ vector", "qdrantApiKeyLabel": "Khóa API Qdrant", "qdrantApiKeyPlaceholder": "Nhập khóa API Qdrant của bạn (tùy chọn)", + "localVectorStoreDirectoryLabel": "Đường dẫn kho lưu trữ vector cục bộ", + "localVectorStoreDirectoryPlaceholder": "Nhập đường dẫn kho lưu trữ vector tùy chỉnh (tùy chọn)", + "localVectorStoreDirectoryDescription": "Đường dẫn để lưu trữ cơ sở dữ liệu vector cục bộ. Nếu trống, sử dụng vị trí mặc định trong globalStorageUri/vector.", "setupConfigLabel": "Cài đặt", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", diff --git a/webview-ui/src/i18n/locales/zh-CN/settings.json b/webview-ui/src/i18n/locales/zh-CN/settings.json index 32e5c96d02..e4b2d1986b 100644 --- a/webview-ui/src/i18n/locales/zh-CN/settings.json +++ b/webview-ui/src/i18n/locales/zh-CN/settings.json @@ -71,10 +71,14 @@ "selectModelPlaceholder": "选择模型", "ollamaUrlLabel": "Ollama URL：", "ollamaBaseUrlLabel": "Ollama 基础 URL", + "vectorStoreProviderLabel": "向量存储提供商", "qdrantUrlLabel": "Qdrant URL", "qdrantKeyLabel": "Qdrant 密钥：", "qdrantApiKeyLabel": "Qdrant API 密钥", "qdrantApiKeyPlaceholder": "输入你的 Qdrant API 密钥（可选）", + "localVectorStoreDirectoryLabel": "本地向量存储路径", + "localVectorStoreDirectoryPlaceholder": "输入自定义向量存储路径（可选）", + "localVectorStoreDirectoryDescription": "存储本地向量数据库的路径。如果为空，将使用 globalStorageUri/vector 的默认位置。", "setupConfigLabel": "设置", "startIndexingButton": "开始", "clearIndexDataButton": "清除索引", diff --git a/webview-ui/src/i18n/locales/zh-TW/settings.json b/webview-ui/src/i18n/locales/zh-TW/settings.json index b8e09bc373..ab32f609d1 100644 --- a/webview-ui/src/i18n/locales/zh-TW/settings.json +++ b/webview-ui/src/i18n/locales/zh-TW/settings.json @@ -82,8 +82,12 @@ "modelPlaceholder": "輸入模型名稱", "selectModel": "選擇模型", "ollamaBaseUrlLabel": "Ollama 基礎 URL", + "vectorStoreProviderLabel": "向量儲存提供者", "qdrantApiKeyLabel": "Qdrant API 金鑰", "qdrantApiKeyPlaceholder": "輸入您的 Qdrant API 金鑰（選用）", + "localVectorStoreDirectoryLabel": "本地向量儲存路徑", + "localVectorStoreDirectoryPlaceholder": "輸入自訂向量儲存路徑（選用）", + "localVectorStoreDirectoryDescription": "儲存本地向量資料庫的路徑。如果為空，將使用 globalStorageUri/vector 的預設位置。", "setupConfigLabel": "設定", "ollamaUrlPlaceholder": "http://localhost:11434", "openAiCompatibleBaseUrlPlaceholder": "https://api.example.com", From 5f28448a1d196f860ebc1967133b09d3a214bfac Mon Sep 17 00:00:00 2001 From: NaccOll Date: Thu, 31 Jul 2025 11:45:53 +0800 Subject: [PATCH 2/7] refactor: replace hardcoded batch sizes with class constants --- src/services/code-index/vector-store/local-vector-store.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/services/code-index/vector-store/local-vector-store.ts b/src/services/code-index/vector-store/local-vector-store.ts index 0c74c6c46b..07af59406a 100644 --- a/src/services/code-index/vector-store/local-vector-store.ts +++ b/src/services/code-index/vector-store/local-vector-store.ts @@ -19,6 +19,9 @@ export class LocalVectorStore implements IVectorStore { private readonly collectionName: string private cachedCollectionId: number | null = null + private readonly UPDATE_BATCH_SIZE = 1000 + private readonly SEARCH_BATCH_SIZE = 10000 + constructor(workspacePath: string, vectorSize: number, dbDirectory: string) { this.vectorSize = vectorSize const basename = path.basename(workspacePath) @@ -181,7 +184,7 @@ export class LocalVectorStore implements IVectorStore { const existingFilesFinal = allFiles const fileIdMap = new Map(existingFilesFinal.map((f) => [f.file_path, f.id])) - const batchSize = 1000 + const batchSize = this.UPDATE_BATCH_SIZE for (let i = 0; i < valids.length; i += batchSize) { const validBatch = valids.slice(i, i + batchSize) @@ -272,7 +275,7 @@ export class LocalVectorStore implements IVectorStore { const cpuCores = os.cpus().length const maxParallelism = Math.max(1, Math.ceil(cpuCores / 2)) - const batchSize = 10000 + const batchSize = this.SEARCH_BATCH_SIZE const totalBatches = Math.ceil(totalCount / batchSize) const actualParallelism = Math.min(maxParallelism, totalBatches) From a055f47a95a248af0b6db56f80f8a66d7c5ad09d Mon Sep 17 00:00:00 2001 From: NaccOll Date: Thu, 31 Jul 2025 11:48:19 +0800 Subject: [PATCH 3/7] chore: update Node.js version in .nvmrc to 22.17.1 --- .nvmrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nvmrc b/.nvmrc index 8320a6d299..7377d130ed 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -22.15.1 +22.17.1 From 679d6ce46bdb328122503cd79c68a1d57661491a Mon Sep 17 00:00:00 2001 From: NaccOll Date: Thu, 31 Jul 2025 11:49:27 +0800 Subject: [PATCH 4/7] fix: limit maximum parallelism to 8 in search processing --- src/services/code-index/vector-store/local-vector-store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/code-index/vector-store/local-vector-store.ts b/src/services/code-index/vector-store/local-vector-store.ts index 07af59406a..f2eb34f7f0 100644 --- a/src/services/code-index/vector-store/local-vector-store.ts +++ b/src/services/code-index/vector-store/local-vector-store.ts @@ -274,7 +274,7 @@ export class LocalVectorStore implements IVectorStore { } const cpuCores = os.cpus().length - const maxParallelism = Math.max(1, Math.ceil(cpuCores / 2)) + const maxParallelism = Math.min(8, Math.max(1, Math.ceil(cpuCores / 2))) const batchSize = this.SEARCH_BATCH_SIZE const totalBatches = Math.ceil(totalCount / batchSize) const actualParallelism = Math.min(maxParallelism, totalBatches) From 82f6f3d3d3e700eb4154723bffccd02266f8425f Mon Sep 17 00:00:00 2001 From: NaccOll Date: Fri, 1 Aug 2025 11:35:51 +0800 Subject: [PATCH 5/7] feat: implement automatic collection resizing based on deleted file count --- .../vector-store/local-vector-store.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/services/code-index/vector-store/local-vector-store.ts b/src/services/code-index/vector-store/local-vector-store.ts index f2eb34f7f0..09021669b3 100644 --- a/src/services/code-index/vector-store/local-vector-store.ts +++ b/src/services/code-index/vector-store/local-vector-store.ts @@ -21,6 +21,10 @@ export class LocalVectorStore implements IVectorStore { private readonly UPDATE_BATCH_SIZE = 1000 private readonly SEARCH_BATCH_SIZE = 10000 + private deletedFileCount = 0 + private lastCleanupTime: Date | null = null + private readonly RESIZE_FILE_THRESHOLD = 50 + private readonly RESIZE_TIME_THRESHOLD_MS = 60 * 60 * 1000 constructor(workspacePath: string, vectorSize: number, dbDirectory: string) { this.vectorSize = vectorSize @@ -409,6 +413,19 @@ export class LocalVectorStore implements IVectorStore { return this.deletePointsByMultipleFilePaths([filePath]) } + private async checkAndResize(): Promise { + const now = new Date() + const shouldResize = + this.deletedFileCount >= this.RESIZE_FILE_THRESHOLD || + (this.lastCleanupTime && now.getTime() - this.lastCleanupTime.getTime() > this.RESIZE_TIME_THRESHOLD_MS) + + if (shouldResize) { + await this.resizeCollection() + this.deletedFileCount = 0 + this.lastCleanupTime = now + } + } + async deletePointsByMultipleFilePaths(filePaths: string[]): Promise { if (filePaths.length === 0) { return @@ -425,7 +442,7 @@ export class LocalVectorStore implements IVectorStore { const workspaceRoot = getWorkspacePath() const normalizedPaths = filePaths.map((fp) => - path.normalize(path.resolve(workspaceRoot, fp)).substring(workspaceRoot.length + 1), + path.normalize(path.isAbsolute(fp) ? path.relative(workspaceRoot, fp) : fp), ) const placeholders = normalizedPaths.map(() => "?").join(",") @@ -443,6 +460,8 @@ export class LocalVectorStore implements IVectorStore { } await db.exec("COMMIT") + this.deletedFileCount += filePaths.length + await this.checkAndResize() } catch (error) { await db.exec("ROLLBACK") console.error("Failed to delete points by file paths:", error) From 4a4356c70afb43a8bf7b56051f741fe850468975 Mon Sep 17 00:00:00 2001 From: NaccOll Date: Fri, 1 Aug 2025 15:37:25 +0800 Subject: [PATCH 6/7] refactor: rename localVectorStoreDirectoryPlaceholder to localVectorStoreDirectory for consistency --- src/services/code-index/config-manager.ts | 19 ++++++++----------- src/services/code-index/interfaces/config.ts | 2 +- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/services/code-index/config-manager.ts b/src/services/code-index/config-manager.ts index b94209cf49..b89121f3d6 100644 --- a/src/services/code-index/config-manager.ts +++ b/src/services/code-index/config-manager.ts @@ -13,7 +13,7 @@ export class CodeIndexConfigManager { private codebaseIndexEnabled: boolean = true private embedderProvider: EmbedderProvider = "openai" private vectorStoreProvider: "local" | "qdrant" = "qdrant" - private localVectorStoreDirectoryPlaceholder?: string + private localVectorStoreDirectory?: string private modelId?: string private modelDimension?: number private openAiOptions?: ApiHandlerOptions @@ -83,7 +83,7 @@ export class CodeIndexConfigManager { // Update instance variables with configuration this.codebaseIndexEnabled = codebaseIndexEnabled ?? true this.vectorStoreProvider = codebaseIndexVectorStoreProvider ?? "qdrant" - this.localVectorStoreDirectoryPlaceholder = codebaseIndexLocalVectorStoreDirectory + this.localVectorStoreDirectory = codebaseIndexLocalVectorStoreDirectory this.qdrantUrl = codebaseIndexQdrantUrl this.qdrantApiKey = qdrantApiKey ?? "" this.searchMinScore = codebaseIndexSearchMinScore @@ -165,7 +165,7 @@ export class CodeIndexConfigManager { configured: this.isConfigured(), embedderProvider: this.embedderProvider, vectorStoreProvider: this.vectorStoreProvider, - localVectorStoreDirectoryPlaceholder: this.localVectorStoreDirectoryPlaceholder, + localVectorStoreDirectory: this.localVectorStoreDirectory, modelId: this.modelId, modelDimension: this.modelDimension, openAiKey: this.openAiOptions?.openAiNativeApiKey ?? "", @@ -272,7 +272,7 @@ export class CodeIndexConfigManager { const prevQdrantUrl = prev?.qdrantUrl ?? "" const prevQdrantApiKey = prev?.qdrantApiKey ?? "" const prevVectorStoreProvider = prev?.vectorStoreProvider ?? "qdrant" - const prevLocalDbPath = prev?.localVectorStoreDirectoryPlaceholder ?? "" + const prevLocalDbPath = prev?.localVectorStoreDirectory ?? "" // 1. Transition from disabled/unconfigured to enabled/configured if ((!prevEnabled || !prevConfigured) && this.codebaseIndexEnabled && nowConfigured) { @@ -306,10 +306,7 @@ export class CodeIndexConfigManager { } // Local DB path change (only affects local vector store) - if ( - this.vectorStoreProvider === "local" && - prevLocalDbPath !== (this.localVectorStoreDirectoryPlaceholder ?? "") - ) { + if (this.vectorStoreProvider === "local" && prevLocalDbPath !== (this.localVectorStoreDirectory ?? "")) { return true } @@ -323,7 +320,7 @@ export class CodeIndexConfigManager { const currentMistralApiKey = this.mistralOptions?.apiKey ?? "" const currentQdrantUrl = this.qdrantUrl ?? "" const currentQdrantApiKey = this.qdrantApiKey ?? "" - const currentLocalDbPath = this.localVectorStoreDirectoryPlaceholder ?? "" + const currentLocalDbPath = this.localVectorStoreDirectory ?? "" if (prevOpenAiKey !== currentOpenAiKey) { return true @@ -404,7 +401,7 @@ export class CodeIndexConfigManager { isConfigured: this.isConfigured(), embedderProvider: this.embedderProvider, vectorStoreProvider: this.vectorStoreProvider ?? "qdrant", - localVectorStoreDirectoryPlaceholder: this.localVectorStoreDirectoryPlaceholder, + localVectorStoreDirectoryPlaceholder: this.localVectorStoreDirectory, modelId: this.modelId, modelDimension: this.modelDimension, openAiOptions: this.openAiOptions, @@ -502,6 +499,6 @@ export class CodeIndexConfigManager { * Gets the current local database path for vector storage */ public get currentLocalDbPath(): string | undefined { - return this.localVectorStoreDirectoryPlaceholder + return this.localVectorStoreDirectory } } diff --git a/src/services/code-index/interfaces/config.ts b/src/services/code-index/interfaces/config.ts index f9b5bb37f2..eaab65f5cf 100644 --- a/src/services/code-index/interfaces/config.ts +++ b/src/services/code-index/interfaces/config.ts @@ -30,7 +30,7 @@ export type PreviousConfigSnapshot = { configured: boolean embedderProvider: EmbedderProvider vectorStoreProvider?: "local" | "qdrant" - localVectorStoreDirectoryPlaceholder?: string + localVectorStoreDirectory?: string modelId?: string modelDimension?: number // Generic dimension property openAiKey?: string From 16219b283addf1d0e0b9f304213b9146a472cffe Mon Sep 17 00:00:00 2001 From: NaccOll Date: Fri, 1 Aug 2025 15:38:35 +0800 Subject: [PATCH 7/7] feat: sqlite optimize and auto_vacuum --- .../__tests__/local-vector-store.spec.ts | 2 +- .../vector-store/local-vector-store.ts | 25 +++---------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts b/src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts index ea9fc504e4..d2314b7893 100644 --- a/src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts +++ b/src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts @@ -1,4 +1,4 @@ -// src/services/code-index/vector-store/__tests__/local-vector-store.spec.ts +// npx vitest run services/code-index/vector-store/__tests__/local-vector-store.spec.ts import { describe, it, expect, beforeEach, vi, afterEach } from "vitest" import { LocalVectorStore } from "../local-vector-store" import type { Payload, VectorStoreSearchResult } from "../../interfaces" diff --git a/src/services/code-index/vector-store/local-vector-store.ts b/src/services/code-index/vector-store/local-vector-store.ts index 09021669b3..87af2ae50d 100644 --- a/src/services/code-index/vector-store/local-vector-store.ts +++ b/src/services/code-index/vector-store/local-vector-store.ts @@ -21,10 +21,6 @@ export class LocalVectorStore implements IVectorStore { private readonly UPDATE_BATCH_SIZE = 1000 private readonly SEARCH_BATCH_SIZE = 10000 - private deletedFileCount = 0 - private lastCleanupTime: Date | null = null - private readonly RESIZE_FILE_THRESHOLD = 50 - private readonly RESIZE_TIME_THRESHOLD_MS = 60 * 60 * 1000 constructor(workspacePath: string, vectorSize: number, dbDirectory: string) { this.vectorSize = vectorSize @@ -57,11 +53,13 @@ export class LocalVectorStore implements IVectorStore { if (!this.db) return this.db.exec(` - PRAGMA journal_mode = WAL; + PRAGMA journal_mode = MEMORY; PRAGMA synchronous = NORMAL; PRAGMA cache_size = 100000; PRAGMA locking_mode = NORMAL; PRAGMA temp_store = MEMORY; + PRAGMA auto_vacuum = INCREMENTAL; + PRAGMA optimize; `) // Create tables if they don't exist await this.db.exec(` @@ -133,7 +131,6 @@ export class LocalVectorStore implements IVectorStore { this.cachedCollectionId = collection.id != null ? Number(collection.id) : null return true } - this.cachedCollectionId = collection.id != null ? Number(collection.id) : null return false } catch (error) { @@ -413,19 +410,6 @@ export class LocalVectorStore implements IVectorStore { return this.deletePointsByMultipleFilePaths([filePath]) } - private async checkAndResize(): Promise { - const now = new Date() - const shouldResize = - this.deletedFileCount >= this.RESIZE_FILE_THRESHOLD || - (this.lastCleanupTime && now.getTime() - this.lastCleanupTime.getTime() > this.RESIZE_TIME_THRESHOLD_MS) - - if (shouldResize) { - await this.resizeCollection() - this.deletedFileCount = 0 - this.lastCleanupTime = now - } - } - async deletePointsByMultipleFilePaths(filePaths: string[]): Promise { if (filePaths.length === 0) { return @@ -460,8 +444,6 @@ export class LocalVectorStore implements IVectorStore { } await db.exec("COMMIT") - this.deletedFileCount += filePaths.length - await this.checkAndResize() } catch (error) { await db.exec("ROLLBACK") console.error("Failed to delete points by file paths:", error) @@ -489,7 +471,6 @@ export class LocalVectorStore implements IVectorStore { if (collectionId != null) { await db.prepare("DELETE FROM vectors WHERE collection_id = ?").run(collectionId) await db.prepare("DELETE FROM files WHERE collection_id = ?").run(collectionId) - await this.resizeCollection() } } catch (error) { console.error("Failed to clear collection:", error)