Skip to content

Commit 66638fc

Browse files
PavelPashovPavelPashov
authored
Add support svs vamana index creation (redis#3025)
* feat(search): add SVS-VAMANA vector index algorithm support - Add VAMANA algorithm with compression and tuning parameters - Include comprehensive test coverage for various configurations - Fix parameter validation to handle falsy values correctly * feat(search): add additional VAMANA compression algorithms - Add LVQ4, LVQ4x4, LVQ4x8, LeanVec4x8, and LeanVec8x8 compression options - Update test to use LeanVec4x8 compression algorithm * chore: update Redis version from 8.2-rc1 to 8.2-rc2-pre
1 parent e2d4b43 commit 66638fc

File tree

10 files changed

+193
-16
lines changed

10 files changed

+193
-16
lines changed

‎.github/workflows/tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
fail-fast: false
2323
matrix:
2424
node-version: ["18", "20", "22"]
25-
redis-version: ["rs-7.4.0-v1", "8.0.2", "8.2-rc1"]
25+
redis-version: ["rs-7.4.0-v1", "8.0.2", "8.2-rc2-pre"]
2626
steps:
2727
- uses: actions/checkout@v4
2828
with:

‎packages/bloom/lib/test-utils.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import RedisBloomModules from '.';
44
export default TestUtils.createFromConfig({
55
dockerImageName: 'redislabs/client-libs-test',
66
dockerImageVersionArgument: 'redis-version',
7-
defaultDockerVersion: '8.2-rc1'
7+
defaultDockerVersion: '8.2-rc2-pre'
88
});
99

1010
export const GLOBAL = {

‎packages/client/lib/sentinel/test-util.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ export class SentinelFramework extends DockerBase {
174174
this.#testUtils = TestUtils.createFromConfig({
175175
dockerImageName: 'redislabs/client-libs-test',
176176
dockerImageVersionArgument: 'redis-version',
177-
defaultDockerVersion: '8.2-rc1'
177+
defaultDockerVersion: '8.2-rc2-pre'
178178
});
179179
this.#nodeMap = new Map<string, ArrayElement<Awaited<ReturnType<SentinelFramework['spawnRedisSentinelNodes']>>>>();
180180
this.#sentinelMap = new Map<string, ArrayElement<Awaited<ReturnType<SentinelFramework['spawnRedisSentinelSentinels']>>>>();

‎packages/client/lib/test-utils.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import RedisBloomModules from '@redis/bloom';
99
const utils = TestUtils.createFromConfig({
1010
dockerImageName: 'redislabs/client-libs-test',
1111
dockerImageVersionArgument: 'redis-version',
12-
defaultDockerVersion: '8.2-rc1'
12+
defaultDockerVersion: '8.2-rc2-pre'
1313
});
1414

1515
export default utils;

‎packages/entraid/lib/test-utils.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { EntraidCredentialsProvider } from './entraid-credentials-provider';
66
export const testUtils = TestUtils.createFromConfig({
77
dockerImageName: 'redislabs/client-libs-test',
88
dockerImageVersionArgument: 'redis-version',
9-
defaultDockerVersion: '8.2-rc1'
9+
defaultDockerVersion: '8.2-rc2-pre'
1010
});
1111

1212
const DEBUG_MODE_ARGS = testUtils.isVersionGreaterThan([7]) ?

‎packages/json/lib/test-utils.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import RedisJSON from '.';
44
export default TestUtils.createFromConfig({
55
dockerImageName: 'redislabs/client-libs-test',
66
dockerImageVersionArgument: 'redis-version',
7-
defaultDockerVersion: '8.2-rc1'
7+
defaultDockerVersion: '8.2-rc2-pre'
88
});
99

1010
export const GLOBAL = {

‎packages/search/lib/commands/CREATE.spec.ts

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { strict as assert } from 'node:assert';
22
import testUtils, { GLOBAL } from '../test-utils';
3-
import CREATE, { SCHEMA_FIELD_TYPE, SCHEMA_TEXT_FIELD_PHONETIC, SCHEMA_VECTOR_FIELD_ALGORITHM, REDISEARCH_LANGUAGE } from './CREATE';
3+
import CREATE, { SCHEMA_FIELD_TYPE, SCHEMA_TEXT_FIELD_PHONETIC, SCHEMA_VECTOR_FIELD_ALGORITHM, REDISEARCH_LANGUAGE, VAMANA_COMPRESSION_ALGORITHM } from './CREATE';
44
import { parseArgs } from '@redis/client/lib/commands/generic-transformers';
55

66
describe('FT.CREATE', () => {
@@ -206,6 +206,33 @@ describe('FT.CREATE', () => {
206206
]
207207
);
208208
});
209+
210+
it('VAMANA algorithm', () => {
211+
assert.deepEqual(
212+
parseArgs(CREATE, 'index', {
213+
field: {
214+
type: SCHEMA_FIELD_TYPE.VECTOR,
215+
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
216+
TYPE: "FLOAT32",
217+
COMPRESSION: VAMANA_COMPRESSION_ALGORITHM.LVQ8,
218+
DIM: 1024,
219+
DISTANCE_METRIC: 'COSINE',
220+
CONSTRUCTION_WINDOW_SIZE: 300,
221+
GRAPH_MAX_DEGREE: 128,
222+
SEARCH_WINDOW_SIZE: 20,
223+
EPSILON: 0.02,
224+
TRAINING_THRESHOLD: 20480,
225+
REDUCE: 512,
226+
}
227+
}),
228+
[
229+
'FT.CREATE', 'index', 'SCHEMA', 'field', 'VECTOR', 'SVS-VAMANA', '20', 'TYPE',
230+
'FLOAT32', 'DIM', '1024', 'DISTANCE_METRIC', 'COSINE', 'COMPRESSION', 'LVQ8',
231+
'CONSTRUCTION_WINDOW_SIZE', '300', 'GRAPH_MAX_DEGREE', '128', 'SEARCH_WINDOW_SIZE', '20',
232+
'EPSILON', '0.02', 'TRAINING_THRESHOLD', '20480', 'REDUCE', '512'
233+
]
234+
);
235+
});
209236
});
210237

211238
describe('GEOSHAPE', () => {
@@ -556,4 +583,87 @@ describe('FT.CREATE', () => {
556583
"OK"
557584
);
558585
}, GLOBAL.SERVERS.OPEN);
586+
587+
testUtils.testWithClientIfVersionWithinRange([[8, 2], 'LATEST'], 'client.ft.create vector svs-vamana', async client => {
588+
assert.equal(
589+
await client.ft.create("index_svs_vamana_min_config", {
590+
field: {
591+
type: SCHEMA_FIELD_TYPE.VECTOR,
592+
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
593+
TYPE: "FLOAT32",
594+
DIM: 768,
595+
DISTANCE_METRIC: 'L2',
596+
},
597+
}),
598+
"OK"
599+
);
600+
601+
assert.equal(
602+
await client.ft.create("index_svs_vamana_no_compression", {
603+
field: {
604+
type: SCHEMA_FIELD_TYPE.VECTOR,
605+
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
606+
TYPE: "FLOAT32",
607+
DIM: 512,
608+
DISTANCE_METRIC: 'L2',
609+
CONSTRUCTION_WINDOW_SIZE: 200,
610+
GRAPH_MAX_DEGREE: 64,
611+
SEARCH_WINDOW_SIZE: 50,
612+
EPSILON: 0.01
613+
},
614+
}),
615+
"OK"
616+
);
617+
618+
assert.equal(
619+
await client.ft.create("index_svs_vamana_compression", {
620+
field: {
621+
type: SCHEMA_FIELD_TYPE.VECTOR,
622+
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
623+
TYPE: "FLOAT32",
624+
COMPRESSION: VAMANA_COMPRESSION_ALGORITHM.LeanVec4x8,
625+
DIM: 1024,
626+
DISTANCE_METRIC: 'COSINE',
627+
CONSTRUCTION_WINDOW_SIZE: 300,
628+
GRAPH_MAX_DEGREE: 128,
629+
SEARCH_WINDOW_SIZE: 20,
630+
EPSILON: 0.02,
631+
TRAINING_THRESHOLD: 20480,
632+
REDUCE: 512,
633+
},
634+
}),
635+
"OK"
636+
);
637+
638+
assert.equal(
639+
await client.ft.create("index_svs_vamana_float16", {
640+
field: {
641+
type: SCHEMA_FIELD_TYPE.VECTOR,
642+
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
643+
TYPE: "FLOAT16",
644+
DIM: 128,
645+
DISTANCE_METRIC: 'IP',
646+
},
647+
}),
648+
"OK"
649+
);
650+
651+
await assert.rejects(
652+
client.ft.create("index_svs_vamana_invalid_config", {
653+
field: {
654+
type: SCHEMA_FIELD_TYPE.VECTOR,
655+
ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.VAMANA,
656+
TYPE: "FLOAT32",
657+
DIM: 2,
658+
DISTANCE_METRIC: 'L2',
659+
CONSTRUCTION_WINDOW_SIZE: 200,
660+
GRAPH_MAX_DEGREE: 64,
661+
SEARCH_WINDOW_SIZE: 50,
662+
EPSILON: 0.01,
663+
// TRAINING_THRESHOLD should error without COMPRESSION
664+
TRAINING_THRESHOLD: 2048
665+
},
666+
}),
667+
)
668+
}, GLOBAL.SERVERS.OPEN);
559669
});

‎packages/search/lib/commands/CREATE.ts

Lines changed: 74 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,11 @@ interface SchemaTagField extends SchemaCommonField<typeof SCHEMA_FIELD_TYPE['TAG
5454

5555
export const SCHEMA_VECTOR_FIELD_ALGORITHM = {
5656
FLAT: 'FLAT',
57-
HNSW: 'HNSW'
57+
HNSW: 'HNSW',
58+
/**
59+
* available since 8.2
60+
*/
61+
VAMANA: 'SVS-VAMANA'
5862
} as const;
5963

6064
export type SchemaVectorFieldAlgorithm = typeof SCHEMA_VECTOR_FIELD_ALGORITHM[keyof typeof SCHEMA_VECTOR_FIELD_ALGORITHM];
@@ -79,6 +83,37 @@ interface SchemaHNSWVectorField extends SchemaVectorField {
7983
EF_RUNTIME?: number;
8084
}
8185

86+
export const VAMANA_COMPRESSION_ALGORITHM = {
87+
LVQ4: 'LVQ4',
88+
LVQ8: 'LVQ8',
89+
LVQ4x4: 'LVQ4x4',
90+
LVQ4x8: 'LVQ4x8',
91+
LeanVec4x8: 'LeanVec4x8',
92+
LeanVec8x8: 'LeanVec8x8'
93+
} as const;
94+
95+
export type VamanaCompressionAlgorithm =
96+
typeof VAMANA_COMPRESSION_ALGORITHM[keyof typeof VAMANA_COMPRESSION_ALGORITHM];
97+
98+
interface SchemaVAMANAVectorField extends SchemaVectorField {
99+
ALGORITHM: typeof SCHEMA_VECTOR_FIELD_ALGORITHM['VAMANA'];
100+
TYPE: 'FLOAT16' | 'FLOAT32';
101+
// VAMANA-specific parameters
102+
COMPRESSION?: VamanaCompressionAlgorithm;
103+
CONSTRUCTION_WINDOW_SIZE?: number;
104+
GRAPH_MAX_DEGREE?: number;
105+
SEARCH_WINDOW_SIZE?: number;
106+
EPSILON?: number;
107+
/**
108+
* applicable only with COMPRESSION
109+
*/
110+
TRAINING_THRESHOLD?: number;
111+
/**
112+
* applicable only with LeanVec COMPRESSION
113+
*/
114+
REDUCE?: number;
115+
}
116+
82117
export const SCHEMA_GEO_SHAPE_COORD_SYSTEM = {
83118
SPHERICAL: 'SPHERICAL',
84119
FLAT: 'FLAT'
@@ -98,6 +133,7 @@ export interface RediSearchSchema {
98133
SchemaTagField |
99134
SchemaFlatVectorField |
100135
SchemaHNSWVectorField |
136+
SchemaVAMANAVectorField |
101137
SchemaGeoShapeField |
102138
SchemaFieldType
103139
);
@@ -142,7 +178,7 @@ export function parseSchema(parser: CommandParser, schema: RediSearchSchema) {
142178
parser.push('NOSTEM');
143179
}
144180

145-
if (fieldOptions.WEIGHT) {
181+
if (fieldOptions.WEIGHT !== undefined) {
146182
parser.push('WEIGHT', fieldOptions.WEIGHT.toString());
147183
}
148184

@@ -197,31 +233,62 @@ export function parseSchema(parser: CommandParser, schema: RediSearchSchema) {
197233
'DISTANCE_METRIC', fieldOptions.DISTANCE_METRIC
198234
);
199235

200-
if (fieldOptions.INITIAL_CAP) {
236+
if (fieldOptions.INITIAL_CAP !== undefined) {
201237
args.push('INITIAL_CAP', fieldOptions.INITIAL_CAP.toString());
202238
}
203239

204240
switch (fieldOptions.ALGORITHM) {
205241
case SCHEMA_VECTOR_FIELD_ALGORITHM.FLAT:
206-
if (fieldOptions.BLOCK_SIZE) {
242+
if (fieldOptions.BLOCK_SIZE !== undefined) {
207243
args.push('BLOCK_SIZE', fieldOptions.BLOCK_SIZE.toString());
208244
}
209245

210246
break;
211247

212248
case SCHEMA_VECTOR_FIELD_ALGORITHM.HNSW:
213-
if (fieldOptions.M) {
249+
if (fieldOptions.M !== undefined) {
214250
args.push('M', fieldOptions.M.toString());
215251
}
216252

217-
if (fieldOptions.EF_CONSTRUCTION) {
253+
if (fieldOptions.EF_CONSTRUCTION !== undefined) {
218254
args.push('EF_CONSTRUCTION', fieldOptions.EF_CONSTRUCTION.toString());
219255
}
220256

221-
if (fieldOptions.EF_RUNTIME) {
257+
if (fieldOptions.EF_RUNTIME !== undefined) {
222258
args.push('EF_RUNTIME', fieldOptions.EF_RUNTIME.toString());
223259
}
224260

261+
break;
262+
263+
case SCHEMA_VECTOR_FIELD_ALGORITHM['VAMANA']:
264+
if (fieldOptions.COMPRESSION) {
265+
args.push('COMPRESSION', fieldOptions.COMPRESSION);
266+
}
267+
268+
if (fieldOptions.CONSTRUCTION_WINDOW_SIZE !== undefined) {
269+
args.push('CONSTRUCTION_WINDOW_SIZE', fieldOptions.CONSTRUCTION_WINDOW_SIZE.toString());
270+
}
271+
272+
if (fieldOptions.GRAPH_MAX_DEGREE !== undefined) {
273+
args.push('GRAPH_MAX_DEGREE', fieldOptions.GRAPH_MAX_DEGREE.toString());
274+
}
275+
276+
if (fieldOptions.SEARCH_WINDOW_SIZE !== undefined) {
277+
args.push('SEARCH_WINDOW_SIZE', fieldOptions.SEARCH_WINDOW_SIZE.toString());
278+
}
279+
280+
if (fieldOptions.EPSILON !== undefined) {
281+
args.push('EPSILON', fieldOptions.EPSILON.toString());
282+
}
283+
284+
if (fieldOptions.TRAINING_THRESHOLD !== undefined) {
285+
args.push('TRAINING_THRESHOLD', fieldOptions.TRAINING_THRESHOLD.toString());
286+
}
287+
288+
if (fieldOptions.REDUCE !== undefined) {
289+
args.push('REDUCE', fieldOptions.REDUCE.toString());
290+
}
291+
225292
break;
226293
}
227294
parser.pushVariadicWithLength(args);

‎packages/search/lib/test-utils.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { RespVersions } from '@redis/client';
55
export default TestUtils.createFromConfig({
66
dockerImageName: 'redislabs/client-libs-test',
77
dockerImageVersionArgument: 'redis-version',
8-
defaultDockerVersion: '8.2-rc1'
8+
defaultDockerVersion: '8.2-rc2-pre'
99
});
1010

1111
export const GLOBAL = {

‎packages/time-series/lib/test-utils.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import TimeSeries from '.';
44
export default TestUtils.createFromConfig({
55
dockerImageName: 'redislabs/client-libs-test',
66
dockerImageVersionArgument: 'redis-version',
7-
defaultDockerVersion: '8.2-rc1'
7+
defaultDockerVersion: '8.2-rc2-pre'
88
});
99

1010
export const GLOBAL = {

0 commit comments

Comments
 (0)