Skip to content

Commit 5dec0a9

Browse files
tsmith023tsmith023
authored
Merge pull request #214 from weaviate/1.27/acorn-support
Support new HNSW filtering strategy config options
2 parents 66114ac + e11fd24 commit 5dec0a9

File tree

10 files changed

+81
-117
lines changed

10 files changed

+81
-117
lines changed

‎.github/workflows/main.yaml‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@ on:
77
pull_request:
88

99
env:
10-
WEAVIATE_124: 1.24.21
11-
WEAVIATE_125: 1.25.8
12-
WEAVIATE_126: 1.26.1
10+
WEAVIATE_124: stable-v1.24-a8b364e
11+
WEAVIATE_125: 1.25.21
12+
WEAVIATE_126: 1.26.7
13+
WEAVIATE_127: 1.27.0
1314

1415
jobs:
1516
checks:
@@ -35,9 +36,10 @@ jobs:
3536
versions: [
3637
{ node: "22.x", weaviate: $WEAVIATE_124},
3738
{ node: "22.x", weaviate: $WEAVIATE_125},
38-
{ node: "18.x", weaviate: $WEAVIATE_126},
39-
{ node: "20.x", weaviate: $WEAVIATE_126},
40-
{ node: "22.x", weaviate: $WEAVIATE_126}
39+
{ node: "22.x", weaviate: $WEAVIATE_126},
40+
{ node: "18.x", weaviate: $WEAVIATE_127},
41+
{ node: "20.x", weaviate: $WEAVIATE_127},
42+
{ node: "22.x", weaviate: $WEAVIATE_127}
4143
]
4244
steps:
4345
- uses: actions/checkout@v3

‎src/collections/config/integration.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ describe('Testing of the collection.config namespace', () => {
6161
dynamicEfMax: 500,
6262
dynamicEfFactor: 8,
6363
vectorCacheMaxObjects: 1000000000000,
64+
filterStrategy: 'sweeping',
6465
flatSearchCutoff: 40000,
6566
distance: 'cosine',
6667
quantizer: undefined,
@@ -115,6 +116,7 @@ describe('Testing of the collection.config namespace', () => {
115116
dynamicEfMax: 500,
116117
dynamicEfFactor: 8,
117118
vectorCacheMaxObjects: 1000000000000,
119+
filterStrategy: 'sweeping',
118120
flatSearchCutoff: 40000,
119121
distance: 'cosine',
120122
quantizer: undefined,
@@ -486,6 +488,7 @@ describe('Testing of the collection.config namespace', () => {
486488
dynamicEfMax: 500,
487489
dynamicEfFactor: 8,
488490
vectorCacheMaxObjects: 1000000000000,
491+
filterStrategy: 'sweeping',
489492
flatSearchCutoff: 40000,
490493
distance: 'cosine',
491494
quantizer: {
@@ -563,6 +566,7 @@ describe('Testing of the collection.config namespace', () => {
563566
dynamicEfMax: 500,
564567
dynamicEfFactor: 8,
565568
vectorCacheMaxObjects: 1000000000000,
569+
filterStrategy: 'sweeping',
566570
flatSearchCutoff: 40000,
567571
distance: 'cosine',
568572
type: 'hnsw',

‎src/collections/config/types/vectorIndex.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ export type VectorIndexConfigHNSW = {
66
dynamicEfFactor: number;
77
efConstruction: number;
88
ef: number;
9+
filterStrategy: VectorIndexFilterStrategy;
910
flatSearchCutoff: number;
1011
maxConnections: number;
1112
quantizer: PQConfig | BQConfig | SQConfig | undefined;
@@ -72,6 +73,8 @@ export type PQEncoderDistribution = 'log-normal' | 'normal';
7273

7374
export type VectorIndexType = 'hnsw' | 'flat' | 'dynamic' | string;
7475

76+
export type VectorIndexFilterStrategy = 'sweeping' | 'acorn';
77+
7578
export type VectorIndexConfig = VectorIndexConfigHNSW | VectorIndexConfigFlat | VectorIndexConfigDynamic;
7679

7780
export type QuantizerConfig = PQConfig | BQConfig | SQConfig;

‎src/collections/config/utils.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import {
4545
VectorIndexConfigFlat,
4646
VectorIndexConfigHNSW,
4747
VectorIndexConfigType,
48+
VectorIndexFilterStrategy,
4849
VectorizerConfig,
4950
} from './types/index.js';
5051

@@ -376,6 +377,7 @@ class ConfigMapping {
376377
dynamicEfFactor: v.dynamicEfFactor,
377378
ef: v.ef,
378379
efConstruction: v.efConstruction,
380+
filterStrategy: exists<VectorIndexFilterStrategy>(v.filterStrategy) ? v.filterStrategy : 'sweeping',
379381
flatSearchCutoff: v.flatSearchCutoff,
380382
maxConnections: v.maxConnections,
381383
quantizer: quantizer,

‎src/collections/configure/types/vectorIndex.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
VectorIndexConfigDynamic,
1010
VectorIndexConfigFlat,
1111
VectorIndexConfigHNSW,
12+
VectorIndexFilterStrategy,
1213
} from '../../config/types/index.js';
1314
import { RecursivePartial } from './base.js';
1415

@@ -56,6 +57,7 @@ export type VectorIndexConfigHNSWUpdate = {
5657
dynamicEfMax?: number;
5758
dynamicEfFactor?: number;
5859
ef?: number;
60+
filterStrategy?: VectorIndexFilterStrategy;
5961
flatSearchCutoff?: number;
6062
quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate;
6163
vectorCacheMaxObjects?: number;

‎src/collections/integration.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -495,7 +495,7 @@ describe('Testing of the collections.create method', () => {
495495
},
496496
vectorizers: weaviate.configure.vectorizer.text2VecContextionary({
497497
vectorIndexConfig: {
498-
name: 'hnsw',
498+
name: 'hnsw' as const,
499499
config: {
500500
cleanupIntervalSeconds: 10,
501501
distance: 'dot',
@@ -504,6 +504,7 @@ describe('Testing of the collections.create method', () => {
504504
dynamicEfMin: 10,
505505
ef: -2,
506506
efConstruction: 100,
507+
filterStrategy: 'acorn',
507508
flatSearchCutoff: 41000,
508509
maxConnections: 72,
509510
quantizer: {
@@ -581,7 +582,9 @@ describe('Testing of the collections.create method', () => {
581582

582583
expect(response.replication.asyncEnabled).toEqual(false);
583584
expect(response.replication.deletionStrategy).toEqual<ReplicationDeletionStrategy>(
584-
'NoAutomatedResolution'
585+
(await cluster.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 25, 0)))
586+
? 'NoAutomatedResolution'
587+
: 'DeleteOnConflict'
585588
);
586589
expect(response.replication.factor).toEqual(2);
587590

@@ -594,6 +597,9 @@ describe('Testing of the collections.create method', () => {
594597
expect(indexConfig.dynamicEfMin).toEqual(10);
595598
expect(indexConfig.ef).toEqual(-2);
596599
expect(indexConfig.efConstruction).toEqual(100);
600+
expect(indexConfig.filterStrategy).toEqual(
601+
(await cluster.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 27, 0))) ? 'sweeping' : 'acorn'
602+
);
597603
expect(indexConfig.flatSearchCutoff).toEqual(41000);
598604
expect(indexConfig.maxConnections).toEqual(72);
599605
expect(quantizer.bitCompression).toEqual(true);

‎src/collections/journey.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ describe('Journey testing of the client using a WCD cluster', () => {
176176
dynamicEfFactor: 8,
177177
ef: -1,
178178
efConstruction: 128,
179+
filterStrategy: 'sweeping',
179180
flatSearchCutoff: 40000,
180181
maxConnections: (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0)))
181182
? 64

‎src/collections/query/integration.test.ts‎

Lines changed: 24 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,12 @@ describe('Testing of the collection.query methods with a simple collection', ()
3535
{
3636
name: 'testProp',
3737
dataType: 'text',
38+
vectorizePropertyName: false,
3839
},
3940
{
4041
name: 'testProp2',
4142
dataType: 'text',
43+
vectorizePropertyName: false,
4244
},
4345
],
4446
vectorizers: weaviate.configure.vectorizer.text2VecContextionary({
@@ -54,8 +56,8 @@ describe('Testing of the collection.query methods with a simple collection', ()
5456
});
5557
return collection.data.insert({
5658
properties: {
57-
testProp: 'test',
58-
testProp2: 'test2',
59+
testProp: 'carrot',
60+
testProp2: 'parsnip',
5961
},
6062
});
6163
});
@@ -65,7 +67,7 @@ describe('Testing of the collection.query methods with a simple collection', ()
6567

6668
it('should fetch an object by its id', async () => {
6769
const object = await collection.query.fetchObjectById(id);
68-
expect(object?.properties.testProp).toEqual('test');
70+
expect(object?.properties.testProp).toEqual('carrot');
6971
expect(object?.uuid).toEqual(id);
7072
});
7173

@@ -87,15 +89,14 @@ describe('Testing of the collection.query methods with a simple collection', ()
8789
});
8890

8991
it('should query with bm25', async () => {
90-
const ret = await collection.query.bm25('test');
92+
const ret = await collection.query.bm25('carrot');
9193
expect(ret.objects.length).toEqual(1);
92-
expect(ret.objects[0].properties.testProp).toEqual('test');
93-
expect(ret.objects[0].properties.testProp2).toEqual('test2');
94+
expect(ret.objects[0].properties.testProp).toEqual('carrot');
9495
expect(ret.objects[0].uuid).toEqual(id);
9596
});
9697

9798
it('should query with bm25 and weighted query properties', async () => {
98-
const ret = await collection.query.bm25('test', {
99+
const ret = await collection.query.bm25('carrot', {
99100
queryProperties: [
100101
{
101102
name: 'testProp',
@@ -105,13 +106,12 @@ describe('Testing of the collection.query methods with a simple collection', ()
105106
],
106107
});
107108
expect(ret.objects.length).toEqual(1);
108-
expect(ret.objects[0].properties.testProp).toEqual('test');
109-
expect(ret.objects[0].properties.testProp2).toEqual('test2');
109+
expect(ret.objects[0].properties.testProp).toEqual('carrot');
110110
expect(ret.objects[0].uuid).toEqual(id);
111111
});
112112

113113
it('should query with bm25 and weighted query properties with a non-generic collection', async () => {
114-
const ret = await client.collections.get(collectionName).query.bm25('test', {
114+
const ret = await client.collections.get(collectionName).query.bm25('carrot', {
115115
queryProperties: [
116116
{
117117
name: 'testProp',
@@ -121,33 +121,30 @@ describe('Testing of the collection.query methods with a simple collection', ()
121121
],
122122
});
123123
expect(ret.objects.length).toEqual(1);
124-
expect(ret.objects[0].properties.testProp).toEqual('test');
125-
expect(ret.objects[0].properties.testProp2).toEqual('test2');
124+
expect(ret.objects[0].properties.testProp).toEqual('carrot');
126125
expect(ret.objects[0].uuid).toEqual(id);
127126
});
128127

129128
it('should query with hybrid', async () => {
130-
const ret = await collection.query.hybrid('test', { limit: 1 });
129+
const ret = await collection.query.hybrid('carrot', { limit: 1 });
131130
expect(ret.objects.length).toEqual(1);
132-
expect(ret.objects[0].properties.testProp).toEqual('test');
133-
expect(ret.objects[0].properties.testProp2).toEqual('test2');
131+
expect(ret.objects[0].properties.testProp).toEqual('carrot');
134132
expect(ret.objects[0].uuid).toEqual(id);
135133
});
136134

137135
it('should query with hybrid and vector', async () => {
138-
const ret = await collection.query.hybrid('test', {
136+
const ret = await collection.query.hybrid('carrot', {
139137
limit: 1,
140138
vector: vector,
141139
});
142140
expect(ret.objects.length).toEqual(1);
143-
expect(ret.objects[0].properties.testProp).toEqual('test');
144-
expect(ret.objects[0].properties.testProp2).toEqual('test2');
141+
expect(ret.objects[0].properties.testProp).toEqual('carrot');
145142
expect(ret.objects[0].uuid).toEqual(id);
146143
});
147144

148145
it('should query with hybrid and near text subsearch', async () => {
149146
const query = () =>
150-
collection.query.hybrid('test', {
147+
collection.query.hybrid('carrot', {
151148
limit: 1,
152149
vector: {
153150
query: 'apple',
@@ -157,7 +154,7 @@ describe('Testing of the collection.query methods with a simple collection', ()
157154
force: 0.9,
158155
},
159156
moveAway: {
160-
concepts: ['test'],
157+
concepts: ['carrot'],
161158
force: 0.1,
162159
},
163160
},
@@ -169,12 +166,11 @@ describe('Testing of the collection.query methods with a simple collection', ()
169166
const ret = await query();
170167
expect(ret.objects.length).toEqual(1);
171168
expect(ret.objects[0].properties.testProp).toEqual('apple');
172-
expect(ret.objects[0].properties.testProp2).toEqual('banana');
173169
});
174170

175171
it('should query with hybrid and near vector subsearch', async () => {
176172
const query = () =>
177-
collection.query.hybrid('test', {
173+
collection.query.hybrid('carrot', {
178174
limit: 1,
179175
vector: {
180176
vector: vector,
@@ -187,31 +183,27 @@ describe('Testing of the collection.query methods with a simple collection', ()
187183
}
188184
const ret = await query();
189185
expect(ret.objects.length).toEqual(1);
190-
expect(ret.objects[0].properties.testProp).toEqual('test');
191-
expect(ret.objects[0].properties.testProp2).toEqual('test2');
186+
expect(ret.objects[0].properties.testProp).toEqual('carrot');
192187
});
193188

194-
it.skip('should query with nearObject', async () => {
189+
it('should query with nearObject', async () => {
195190
const ret = await collection.query.nearObject(id, { limit: 1 });
196191
expect(ret.objects.length).toEqual(1);
197-
expect(ret.objects[0].properties.testProp).toEqual('test');
198-
expect(ret.objects[0].properties.testProp2).toEqual('test2');
192+
expect(ret.objects[0].properties.testProp).toEqual('carrot');
199193
expect(ret.objects[0].uuid).toEqual(id);
200194
});
201195

202196
it('should query with nearText', async () => {
203-
const ret = await collection.query.nearText(['test'], { limit: 1 });
197+
const ret = await collection.query.nearText(['carrot'], { limit: 1 });
204198
expect(ret.objects.length).toEqual(1);
205-
expect(ret.objects[0].properties.testProp).toEqual('test');
206-
expect(ret.objects[0].properties.testProp2).toEqual('test2');
199+
expect(ret.objects[0].properties.testProp).toEqual('carrot');
207200
expect(ret.objects[0].uuid).toEqual(id);
208201
});
209202

210203
it('should query with nearVector', async () => {
211204
const ret = await collection.query.nearVector(vector, { limit: 1 });
212205
expect(ret.objects.length).toEqual(1);
213-
expect(ret.objects[0].properties.testProp).toEqual('test');
214-
expect(ret.objects[0].properties.testProp2).toEqual('test2');
206+
expect(ret.objects[0].properties.testProp).toEqual('carrot');
215207
expect(ret.objects[0].uuid).toEqual(id);
216208
});
217209
});

‎src/graphql/journey.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1659,7 +1659,7 @@ describe('query cluster with consistency level', () => {
16591659
});
16601660
});
16611661

1662-
describe('query with group by', () => {
1662+
describe.skip('query with group by SKIPPED BECAUSE OF XREFS RETURN OPTIMISATION BUG', () => {
16631663
let client: WeaviateClient;
16641664

16651665
beforeEach(() => {

0 commit comments

Comments
 (0)