Skip to content

Commit 3b6f9f3

Browse files
authored
Merge pull request #123 from weaviate/feat/quantizer-argument
Add quantizer parameter to VectorConfig factory helper
2 parents 521cde1 + 75314cf commit 3b6f9f3

File tree

15 files changed

+306
-27
lines changed

15 files changed

+306
-27
lines changed

src/Weaviate.Client.Tests/Integration/TestAuth.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public async Task TestNoAuthProvided()
7878

7979
var client = Connect.Local(hostname: "localhost", restPort: OKTA_PORT_CC);
8080

81-
await Assert.ThrowsAsync<WeaviateException>(async () =>
81+
await Assert.ThrowsAnyAsync<WeaviateServerException>(async () =>
8282
{
8383
await client.Collections.List().ToListAsync(TestContext.Current.CancellationToken);
8484
});

src/Weaviate.Client.Tests/Integration/TestTenant.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,7 @@ public async Task TenantsCreateWithReadOnlyActivityStatus(object tenants)
516516
multiTenancyConfig: Configure.MultiTenancy(enabled: true)
517517
);
518518

519-
await Assert.ThrowsAsync<WeaviateException>(async () =>
519+
await Assert.ThrowsAnyAsync<WeaviateServerException>(async () =>
520520
{
521521
if (tenants is Tenant t)
522522
await collectionClient.Tenants.Add(t);
@@ -557,7 +557,7 @@ public async Task TenantsUpdateWithReadOnlyActivityStatus(object tenants)
557557
multiTenancyConfig: Configure.MultiTenancy(enabled: true)
558558
);
559559

560-
await Assert.ThrowsAsync<WeaviateException>(async () =>
560+
await Assert.ThrowsAnyAsync<WeaviateServerException>(async () =>
561561
{
562562
if (tenants is Tenant t)
563563
await collectionClient.Tenants.Update(t);

src/Weaviate.Client.Tests/Unit/TestVectorIndexConfig.cs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,30 @@ string expectedQuantizer
2929
Assert.Equal(expectedQuantizer, hnsw?.Quantizer.Type);
3030
Assert.Null(hnsw!.MultiVector);
3131
}
32+
33+
[Fact]
34+
public void VectorIndexConfig_Sets_Quantizer_When_Provided()
35+
{
36+
// Arrange
37+
var quantizer = new VectorIndex.Quantizers.BQ();
38+
39+
var config = new VectorIndex.HNSW { Ef = 100, MaxConnections = 16 };
40+
41+
// Act & Assert
42+
var vectorConfig = Configure.Vectors.SelfProvided(
43+
name: "regular",
44+
indexConfig: config,
45+
quantizerConfig: quantizer
46+
);
47+
48+
Assert.Null(config.Quantizer);
49+
50+
Assert.NotNull(vectorConfig);
51+
Assert.Equal("regular", vectorConfig.Name);
52+
Assert.IsType<VectorIndex.HNSW>(vectorConfig.VectorIndexConfig);
53+
Assert.NotNull((vectorConfig.VectorIndexConfig as VectorIndex.HNSW)?.Quantizer);
54+
Assert.IsType<VectorIndex.Quantizers.BQ>(
55+
(vectorConfig.VectorIndexConfig as VectorIndex.HNSW)?.Quantizer
56+
);
57+
}
3258
}

src/Weaviate.Client.Tests/Unit/TestVectorizers.cs

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,67 @@ namespace Weaviate.Client.Tests.Unit;
77

88
public partial class VectorConfigListTests
99
{
10+
[Fact]
11+
public void Throws_When_FlatIndex_With_NonBQ_Quantizer()
12+
{
13+
var flat = new VectorIndex.Flat();
14+
var pq = new Quantizers.PQ
15+
{
16+
Encoder = new Quantizers.PQ.EncoderConfig
17+
{
18+
Distribution = Quantizers.DistributionType.Normal,
19+
Type = Quantizers.EncoderType.Kmeans,
20+
},
21+
BitCompression = false,
22+
Centroids = 256,
23+
Segments = 8,
24+
TrainingLimit = 1000,
25+
};
26+
Assert.Throws<WeaviateClientException>(() =>
27+
Configure.Vectors.SelfProvided("flat-pq", flat, pq)
28+
);
29+
}
30+
31+
[Fact]
32+
public void Throws_When_FlatIndex_Already_Has_Quantizer()
33+
{
34+
var flat = new VectorIndex.Flat
35+
{
36+
Quantizer = new Quantizers.BQ { Cache = true, RescoreLimit = 10 },
37+
};
38+
var bq = new Quantizers.BQ { Cache = false, RescoreLimit = 5 };
39+
Assert.Throws<WeaviateClientException>(() =>
40+
Configure.Vectors.SelfProvided("flat-bq", flat, bq)
41+
);
42+
}
43+
44+
[Fact]
45+
public void Throws_When_HNSW_Already_Has_Quantizer()
46+
{
47+
var hnsw = new VectorIndex.HNSW
48+
{
49+
Quantizer = new Quantizers.BQ { Cache = true, RescoreLimit = 10 },
50+
};
51+
var bq = new Quantizers.BQ { Cache = false, RescoreLimit = 5 };
52+
Assert.Throws<WeaviateClientException>(() =>
53+
Configure.Vectors.SelfProvided("hnsw-bq", hnsw, bq)
54+
);
55+
}
56+
57+
[Fact]
58+
public void Throws_When_DynamicIndex_With_Quantizer()
59+
{
60+
var dynamic = new VectorIndex.Dynamic
61+
{
62+
Hnsw = new VectorIndex.HNSW(),
63+
Flat = new VectorIndex.Flat(),
64+
};
65+
var bq = new Quantizers.BQ { Cache = false, RescoreLimit = 5 };
66+
Assert.Throws<WeaviateClientException>(() =>
67+
Configure.Vectors.SelfProvided("dynamic-bq", dynamic, bq)
68+
);
69+
}
70+
1071
[Fact]
1172
public void NamedVectorInitialization()
1273
{

src/Weaviate.Client/Configure/Vectorizer.Multivector.cs

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Weaviate.Client.Models;
2+
using static Weaviate.Client.Models.VectorIndexConfig;
23

34
namespace Weaviate.Client;
45

@@ -8,17 +9,23 @@ public static class MultiVectors
89
{
910
public static VectorConfig SelfProvided(
1011
string name = "default",
11-
VectorIndex.HNSW? indexConfig = null
12+
VectorIndex.HNSW? indexConfig = null,
13+
QuantizerConfig? quantizerConfig = null
1214
)
1315
{
14-
return new VectorConfigBuilder(new Vectorizer.SelfProvided()).New(name, indexConfig);
16+
return new VectorConfigBuilder(new Vectorizer.SelfProvided()).New(
17+
name,
18+
indexConfig,
19+
quantizerConfig
20+
);
1521
}
1622

1723
public class VectorConfigBuilder(VectorizerConfig Config)
1824
{
1925
public VectorConfig New(
2026
string name,
2127
VectorIndex.HNSW? indexConfig = null,
28+
QuantizerConfig? quantizerConfig = null,
2229
params string[] sourceProperties
2330
)
2431
{
@@ -29,13 +36,28 @@ params string[] sourceProperties
2936

3037
indexConfig.MultiVector ??= new VectorIndexConfig.MultiVectorConfig();
3138

39+
if (quantizerConfig is not null && indexConfig.Quantizer is not null)
40+
{
41+
throw new WeaviateClientException(
42+
null,
43+
new InvalidOperationException(
44+
"Quantizer is already set on the indexConfig. Please provide either the quantizerConfig or set it on the indexConfig, not both."
45+
)
46+
);
47+
}
48+
3249
return new(
3350
name,
3451
vectorizer: Config with
3552
{
3653
SourceProperties = sourceProperties,
3754
},
38-
vectorIndexConfig: indexConfig
55+
vectorIndexConfig: quantizerConfig is null
56+
? indexConfig
57+
: indexConfig with
58+
{
59+
Quantizer = quantizerConfig,
60+
}
3961
);
4062
}
4163
}

src/Weaviate.Client/Configure/Vectorizer.cs

Lines changed: 152 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,171 @@ public static class Vectors
88
{
99
public static VectorConfig SelfProvided(
1010
string name = "default",
11-
VectorIndexConfig? indexConfig = null
12-
) => new VectorConfigBuilder(new Vectorizer.SelfProvided()).New(name, indexConfig);
11+
VectorIndexConfig? indexConfig = null,
12+
VectorIndexConfig.QuantizerConfig? quantizerConfig = null
13+
)
14+
{
15+
var builder = new VectorConfigBuilder(new Vectorizer.SelfProvided());
16+
return indexConfig switch
17+
{
18+
VectorIndex.HNSW hnsw => builder.New(name, hnsw, quantizerConfig),
19+
VectorIndex.Flat flat => builder.New(
20+
name,
21+
flat,
22+
quantizerConfig is not null
23+
? quantizerConfig as VectorIndex.Quantizers.BQ
24+
?? throw new WeaviateClientException(
25+
"Flat index supports only BQ quantization."
26+
)
27+
: null
28+
),
29+
VectorIndex.Dynamic dynamic => quantizerConfig is null
30+
? builder.New(name, dynamic)
31+
: throw new WeaviateClientException(
32+
"Dynamic Index must specify quantizers in their respective Vector Index Configurations."
33+
),
34+
null => builder.New(
35+
name,
36+
(VectorIndex.HNSW?)null,
37+
(VectorIndexConfig.QuantizerConfig?)null
38+
),
39+
_ => throw new WeaviateClientException(
40+
$"Unsupported VectorIndexConfig type: {indexConfig.GetType().Name}"
41+
),
42+
};
43+
}
1344

1445
public class VectorConfigBuilder(VectorizerConfig Config)
1546
{
47+
public VectorConfig New(string name = "default", params string[] sourceProperties) =>
48+
new(
49+
name,
50+
vectorizer: Config with
51+
{
52+
SourceProperties = sourceProperties,
53+
},
54+
vectorIndexConfig: null
55+
);
56+
1657
public VectorConfig New(
17-
string name = "default",
18-
VectorIndexConfig? indexConfig = null,
58+
string name,
59+
VectorIndex.HNSW? indexConfig,
60+
VectorIndexConfig.QuantizerConfig? quantizerConfig = null,
1961
params string[] sourceProperties
2062
) =>
2163
new(
22-
name,
64+
name: string.IsNullOrEmpty(name) ? "default" : name,
65+
vectorizer: Config with
66+
{
67+
SourceProperties = sourceProperties,
68+
},
69+
vectorIndexConfig: EnrichVectorIndexConfig(indexConfig, quantizerConfig)
70+
);
71+
72+
public VectorConfig New(
73+
string name,
74+
VectorIndex.Flat? indexConfig,
75+
VectorIndex.Quantizers.BQ? quantizerConfig = null,
76+
params string[] sourceProperties
77+
) =>
78+
new(
79+
name: string.IsNullOrEmpty(name) ? "default" : name,
80+
vectorizer: Config with
81+
{
82+
SourceProperties = sourceProperties,
83+
},
84+
vectorIndexConfig: EnrichVectorIndexConfig(indexConfig, quantizerConfig)
85+
);
86+
87+
public VectorConfig New(
88+
string name,
89+
VectorIndex.Dynamic? indexConfig,
90+
params string[] sourceProperties
91+
) =>
92+
new(
93+
name: string.IsNullOrEmpty(name) ? "default" : name,
2394
vectorizer: Config with
2495
{
2596
SourceProperties = sourceProperties,
2697
},
2798
vectorIndexConfig: indexConfig
2899
);
100+
101+
/// <summary>
102+
/// Enriches the provided <see cref="VectorIndexConfig"/> instance with the specified quantizer configuration,
103+
/// if applicable. The method updates the quantizer property of the index configuration based on its concrete type.
104+
/// </summary>
105+
/// <param name="indexConfig">
106+
/// The vector index configuration to enrich. If <c>null</c>, the method returns <c>null</c>.
107+
/// </param>
108+
/// <param name="quantizerConfig">
109+
/// The quantizer configuration to apply. If <c>null</c>, the original <paramref name="indexConfig"/> is returned unchanged.
110+
/// </param>
111+
/// <returns>
112+
/// The enriched <see cref="VectorIndexConfig"/> instance with the quantizer configuration applied, or the original
113+
/// <paramref name="indexConfig"/> if no enrichment was possible.
114+
/// </returns>
115+
private static VectorIndexConfig? EnrichVectorIndexConfig(
116+
VectorIndexConfig? indexConfig,
117+
VectorIndexConfig.QuantizerConfig? quantizerConfig
118+
)
119+
{
120+
if (indexConfig is null)
121+
return null;
122+
123+
if (quantizerConfig is null)
124+
return indexConfig;
125+
126+
if (indexConfig is VectorIndex.HNSW hnsw)
127+
{
128+
if (hnsw.Quantizer != null)
129+
{
130+
throw new WeaviateClientException(
131+
"HNSW index already has a quantizer configured. Overwriting is not allowed."
132+
);
133+
}
134+
135+
return hnsw with
136+
{
137+
Quantizer = quantizerConfig,
138+
};
139+
}
140+
141+
if (indexConfig is VectorIndex.Flat flat)
142+
{
143+
if (flat.Quantizer != null)
144+
{
145+
throw new WeaviateClientException(
146+
"Flat index already has a quantizer configured. Overwriting is not allowed."
147+
);
148+
}
149+
150+
// Only set the Quantizer if it's of type BQ, as Flat supports only BQ quantization.
151+
if (quantizerConfig is VectorIndex.Quantizers.BQ bq)
152+
{
153+
flat.Quantizer = bq;
154+
}
155+
else
156+
{
157+
throw new WeaviateClientException(
158+
"Flat index supports only BQ quantization. Provided quantizer is of type: "
159+
+ quantizerConfig.GetType().Name
160+
);
161+
}
162+
return flat;
163+
}
164+
165+
// Handle the case where the index configuration is of type Dynamic,
166+
// which may contain both HNSW and Flat sub-configurations.
167+
if (indexConfig is VectorIndex.Dynamic)
168+
{
169+
throw new WeaviateClientException(
170+
"Dynamic Index must specify quantizers in their respective Vector Index Configurations."
171+
);
172+
}
173+
174+
return indexConfig;
175+
}
29176
}
30177

31178
public static VectorConfigBuilder Img2VecNeural(string[] imageFields) =>

src/Weaviate.Client/DataClient.cs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,9 @@ params BatchInsertRequest<TData>[] requests
170170
new BatchInsertResponse(
171171
Index: r,
172172
dictUuid.TryGetValue(r, out Guid uuid) ? uuid : (Guid?)null,
173-
dictErr.TryGetValue(r, out string? error) ? new WeaviateException(error) : null
173+
dictErr.TryGetValue(r, out string? error)
174+
? new WeaviateClientException(error)
175+
: null
174176
)
175177
);
176178
}
@@ -275,7 +277,9 @@ public async Task<BatchReferenceReturn> ReferenceAddMany(params DataReference[]
275277
{
276278
var errors = entry.Result?.Errors?.Error ?? Enumerable.Empty<Error>();
277279

278-
return errors.Select(e => new WeaviateException(e.Message)).ToArray();
280+
return errors
281+
.Select(e => new WeaviateClientException(e.Message) as WeaviateException)
282+
.ToArray();
279283
}
280284
);
281285

0 commit comments

Comments
 (0)