From 4f3a4c56c46dc4ed9637b8fa600f05c4b72c96d0 Mon Sep 17 00:00:00 2001 From: daniel-lxs Date: Mon, 18 Aug 2025 08:50:20 -0500 Subject: [PATCH 1/5] feat: enable on-disk storage for Qdrant vectors and HNSW index - Configure vectors with on_disk: true to reduce memory usage - Configure HNSW index with on_disk: true for low memory footprint - Improve precision with m=64 and ef_construct=512 parameters - Increase hnsw_ef from 128 to 256 for better search precision - Remove error throwing in deletePointsByMultipleFilePaths to prevent disruption --- .../code-index/vector-store/qdrant-client.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 50f39666c4..b83092b28a 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -155,6 +155,12 @@ export class QdrantVectorStore implements IVectorStore { vectors: { size: this.vectorSize, distance: this.DISTANCE_METRIC, + on_disk: true, // Store vectors on disk for low memory usage + }, + hnsw_config: { + m: 64, // Increased for better precision + ef_construct: 512, // Increased for better precision during index construction + on_disk: true, // Store HNSW index on disk for low memory usage }, }) created = true @@ -244,6 +250,12 @@ export class QdrantVectorStore implements IVectorStore { vectors: { size: this.vectorSize, distance: this.DISTANCE_METRIC, + on_disk: true, // Store vectors on disk for low memory usage + }, + hnsw_config: { + m: 64, // Increased for better precision + ef_construct: 512, // Increased for better precision during index construction + on_disk: true, // Store HNSW index on disk for low memory usage }, }) console.log(`[QdrantVectorStore] Successfully created new collection ${this.collectionName}`) @@ -404,7 +416,7 @@ export class QdrantVectorStore implements IVectorStore { score_threshold: minScore ?? DEFAULT_SEARCH_MIN_SCORE, limit: maxResults ?? DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, // Increased from 128 for better search precision with on-disk storage exact: false, }, with_payload: { @@ -491,8 +503,6 @@ export class QdrantVectorStore implements IVectorStore { // Include first few file paths for debugging (avoid logging too many) samplePaths: filePaths.slice(0, 3), }) - - throw error } } From 85a70f3f5161ca8e149d6f7b28d4671f7f736db5 Mon Sep 17 00:00:00 2001 From: daniel-lxs Date: Mon, 18 Aug 2025 09:31:25 -0500 Subject: [PATCH 2/5] test: update Qdrant tests to match new on-disk storage configuration - Update test expectations for createCollection to include on_disk: true for vectors - Add hnsw_config expectations with m: 64, ef_construct: 512, and on_disk: true - Update search test expectations to use hnsw_ef: 256 instead of 128 These changes align the tests with the PR's implementation of on-disk storage for reduced memory usage and improved precision. --- .../__tests__/qdrant-client.spec.ts | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index 822832d17c..643cc94278 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -532,6 +532,12 @@ describe("QdrantVectorStore", () => { vectors: { size: mockVectorSize, distance: "Cosine", // Assuming 'Cosine' is the DISTANCE_METRIC + on_disk: true, + }, + hnsw_config: { + m: 64, + ef_construct: 512, + on_disk: true, }, }) expect(mockQdrantClientInstance.deleteCollection).not.toHaveBeenCalled() @@ -610,6 +616,12 @@ describe("QdrantVectorStore", () => { vectors: { size: mockVectorSize, // Should use the new, correct vector size distance: "Cosine", + on_disk: true, + }, + hnsw_config: { + m: 64, + ef_construct: 512, + on_disk: true, }, }) @@ -903,6 +915,12 @@ describe("QdrantVectorStore", () => { vectors: { size: newVectorSize, // Should create with new 768 dimensions distance: "Cosine", + on_disk: true, + }, + hnsw_config: { + m: 64, + ef_construct: 512, + on_disk: true, }, }) expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(5) @@ -1248,7 +1266,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1299,7 +1317,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1325,7 +1343,7 @@ describe("QdrantVectorStore", () => { score_threshold: customMinScore, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1349,7 +1367,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: customMaxResults, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1496,7 +1514,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1561,7 +1579,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1587,7 +1605,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1611,7 +1629,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1635,7 +1653,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1659,7 +1677,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1690,7 +1708,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { @@ -1721,7 +1739,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 128, + hnsw_ef: 256, exact: false, }, with_payload: { From 5c5a3b8cff5d0e6361724354e1b4b48a606b9b6a Mon Sep 17 00:00:00 2001 From: daniel-lxs Date: Mon, 25 Aug 2025 11:57:26 -0500 Subject: [PATCH 3/5] fix: revert hnsw_ef parameter to 128 for search precision with on-disk storage --- src/services/code-index/vector-store/qdrant-client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index b83092b28a..3af30775fe 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -416,7 +416,7 @@ export class QdrantVectorStore implements IVectorStore { score_threshold: minScore ?? DEFAULT_SEARCH_MIN_SCORE, limit: maxResults ?? DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, // Increased from 128 for better search precision with on-disk storage + hnsw_ef: 128, exact: false, }, with_payload: { From 1b6ab34b2970f45cb5f0a786f413c0427ae79db8 Mon Sep 17 00:00:00 2001 From: daniel-lxs Date: Mon, 25 Aug 2025 12:23:40 -0500 Subject: [PATCH 4/5] chore: remove change-description comments from Qdrant configuration Remove inline comments that describe what was changed/increased/added as they pollute the codebase. The configuration values remain unchanged. --- .../code-index/vector-store/qdrant-client.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 3af30775fe..f18f388323 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -155,12 +155,12 @@ export class QdrantVectorStore implements IVectorStore { vectors: { size: this.vectorSize, distance: this.DISTANCE_METRIC, - on_disk: true, // Store vectors on disk for low memory usage + on_disk: true, }, hnsw_config: { - m: 64, // Increased for better precision - ef_construct: 512, // Increased for better precision during index construction - on_disk: true, // Store HNSW index on disk for low memory usage + m: 64, + ef_construct: 512, + on_disk: true, }, }) created = true @@ -250,12 +250,12 @@ export class QdrantVectorStore implements IVectorStore { vectors: { size: this.vectorSize, distance: this.DISTANCE_METRIC, - on_disk: true, // Store vectors on disk for low memory usage + on_disk: true, }, hnsw_config: { - m: 64, // Increased for better precision - ef_construct: 512, // Increased for better precision during index construction - on_disk: true, // Store HNSW index on disk for low memory usage + m: 64, + ef_construct: 512, + on_disk: true, }, }) console.log(`[QdrantVectorStore] Successfully created new collection ${this.collectionName}`) From 74fe2d17950ba3d4a2e3f84c713d9165c7b53f70 Mon Sep 17 00:00:00 2001 From: daniel-lxs Date: Mon, 25 Aug 2025 16:51:54 -0500 Subject: [PATCH 5/5] fix: update test expectations for reduced hnsw_ef parameter The PR changed hnsw_ef from 256 to 128 for improved search precision, but the tests were still expecting the old value. This commit updates all test expectations to match the new hnsw_ef value of 128. --- .../__tests__/qdrant-client.spec.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index 643cc94278..8947c2f3e7 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -1266,7 +1266,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1317,7 +1317,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1343,7 +1343,7 @@ describe("QdrantVectorStore", () => { score_threshold: customMinScore, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1367,7 +1367,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: customMaxResults, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1514,7 +1514,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1579,7 +1579,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1605,7 +1605,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1629,7 +1629,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1653,7 +1653,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1677,7 +1677,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1708,7 +1708,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: { @@ -1739,7 +1739,7 @@ describe("QdrantVectorStore", () => { score_threshold: DEFAULT_SEARCH_MIN_SCORE, limit: DEFAULT_MAX_SEARCH_RESULTS, params: { - hnsw_ef: 256, + hnsw_ef: 128, exact: false, }, with_payload: {