-
Notifications
You must be signed in to change notification settings - Fork 2.6k
feat: Enable on-disk storage for Qdrant vectors and HNSW index #7182
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
4f3a4c5
85a70f3
5c5a3b8
1b6ab34
74fe2d1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -155,6 +155,12 @@ export class QdrantVectorStore implements IVectorStore { | |
| vectors: { | ||
| size: this.vectorSize, | ||
| distance: this.DISTANCE_METRIC, | ||
| on_disk: true, // Store vectors on disk for low memory usage | ||
daniel-lxs marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| }, | ||
| hnsw_config: { | ||
| m: 64, // Increased for better precision | ||
|
||
| ef_construct: 512, // Increased for better precision during index construction | ||
| on_disk: true, // Store HNSW index on disk for low memory usage | ||
| }, | ||
| }) | ||
| created = true | ||
|
|
@@ -244,6 +250,12 @@ export class QdrantVectorStore implements IVectorStore { | |
| vectors: { | ||
| size: this.vectorSize, | ||
| distance: this.DISTANCE_METRIC, | ||
| on_disk: true, // Store vectors on disk for low memory usage | ||
| }, | ||
| hnsw_config: { | ||
| m: 64, // Increased for better precision | ||
| ef_construct: 512, // Increased for better precision during index construction | ||
| on_disk: true, // Store HNSW index on disk for low memory usage | ||
| }, | ||
| }) | ||
| console.log(`[QdrantVectorStore] Successfully created new collection ${this.collectionName}`) | ||
|
|
@@ -404,7 +416,7 @@ export class QdrantVectorStore implements IVectorStore { | |
| score_threshold: minScore ?? DEFAULT_SEARCH_MIN_SCORE, | ||
| limit: maxResults ?? DEFAULT_MAX_SEARCH_RESULTS, | ||
| params: { | ||
| hnsw_ef: 128, | ||
| hnsw_ef: 256, // Increased from 128 for better search precision with on-disk storage | ||
daniel-lxs marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| exact: false, | ||
| }, | ||
| with_payload: { | ||
|
|
@@ -491,8 +503,6 @@ export class QdrantVectorStore implements IVectorStore { | |
| // Include first few file paths for debugging (avoid logging too many) | ||
| samplePaths: filePaths.slice(0, 3), | ||
| }) | ||
|
|
||
| throw error | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Removing the here silently swallows errors, which might hide important issues. While preventing disruption is good, consider:
For example: |
||
| } | ||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.