Skip to content

Commit c1e65f5

Browse files
bevzzzbevzzz
authored
Merge pull request #492 from weaviate/v6-vectorizers
v6: Vectorizer modules config
2 parents 08bd1ac + 572f369 commit c1e65f5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+5818
-288
lines changed

‎pom.xml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -52,26 +52,26 @@
5252
<maven.compiler.source>17</maven.compiler.source>
5353
<maven.compiler.target>17</maven.compiler.target>
5454
<maven.compiler.release>17</maven.compiler.release>
55-
<lombok.version>1.18.38</lombok.version>
56-
<gson.version>2.13.1</gson.version>
57-
<httpclient.version>5.5</httpclient.version>
58-
<lang3.version>3.18.0</lang3.version>
55+
<lombok.version>1.18.42</lombok.version>
56+
<gson.version>2.13.2</gson.version>
57+
<httpclient.version>5.5.1</httpclient.version>
58+
<lang3.version>3.19.0</lang3.version>
5959
<junit.version>5.13.4</junit.version>
6060
<testcontainers.version>1.21.3</testcontainers.version>
61-
<assertj-core.version>3.27.4</assertj-core.version>
61+
<assertj-core.version>3.27.6</assertj-core.version>
6262
<jparams.version>1.0.4</jparams.version>
63-
<mockito.version>5.19.0</mockito.version>
64-
<slf4j.version>1.7.36</slf4j.version>
63+
<mockito.version>5.20.0</mockito.version>
64+
<slf4j.version>2.0.17</slf4j.version>
6565
<logback.version>1.5.18</logback.version>
6666
<mock-server.version>5.14.0</mock-server.version>
67-
<jackson.version>2.19.2</jackson.version>
68-
<oauth2-oidc-sdk.version>11.27.1</oauth2-oidc-sdk.version>
67+
<jackson.version>2.20</jackson.version>
68+
<oauth2-oidc-sdk.version>11.30</oauth2-oidc-sdk.version>
6969
<mock-server.version>5.15.0</mock-server.version>
70-
<protobuf.java.version>4.32.0</protobuf.java.version>
71-
<protobuf.java-util.version>4.32.0</protobuf.java-util.version>
72-
<grpc-netty-shaded.version>1.75.0</grpc-netty-shaded.version>
73-
<grpc-protobuf.version>1.75.0</grpc-protobuf.version>
74-
<grpc-stub.version>1.75.0</grpc-stub.version>
70+
<protobuf.java.version>4.33.0</protobuf.java.version>
71+
<protobuf.java-util.version>4.33.0</protobuf.java-util.version>
72+
<grpc-netty-shaded.version>1.76.0</grpc-netty-shaded.version>
73+
<grpc-protobuf.version>1.76.0</grpc-protobuf.version>
74+
<grpc-stub.version>1.76.0</grpc-stub.version>
7575
<annotations-api.version>6.0.53</annotations-api.version>
7676
</properties>
7777

‎src/it/java/io/weaviate/containers/Container.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
public class Container {
1616
public static final Weaviate WEAVIATE = Weaviate.createDefault();
17-
public static final Contextionary CONTEXTIONARY = Contextionary.createDefault();
17+
public static final Model2Vec MODEL2VEC = Model2Vec.createDefault();
1818
public static final Img2VecNeural IMG2VEC_NEURAL = Img2VecNeural.createDefault();
1919
public static final MinIo MINIO = MinIo.createDefault();
2020

‎src/it/java/io/weaviate/containers/Contextionary.java‎

Lines changed: 0 additions & 46 deletions
This file was deleted.

‎src/it/java/io/weaviate/containers/MinIo.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ public class MinIo extends MinIOContainer {
77
public static final String ACCESS_KEY = "minioadmin";
88
public static final String SECRET_KEY = "minioadmin";
99

10+
public static final String HOST_NAME = "minio";
11+
public static final String URL = HOST_NAME + ":9000";
12+
1013
static MinIo createDefault() {
1114
return new MinIo();
1215
}

‎src/it/java/io/weaviate/containers/Model2Vec.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package io.weaviate.containers;
2+
3+
import org.testcontainers.containers.GenericContainer;
4+
5+
import io.weaviate.client6.v1.api.collections.VectorConfig;
6+
7+
public class Model2Vec extends GenericContainer<Model2Vec> {
8+
public static final String VERSION = "minishlab-potion-base-4M";
9+
public static final String DOCKER_IMAGE = "cr.weaviate.io/semitechnologies/model2vec-inference";
10+
public static final String MODULE = VectorConfig.Kind.TEXT2VEC_MODEL2VEC.jsonValue();
11+
12+
public static final String HOST_NAME = "model2vec";
13+
public static final String URL = HOST_NAME + ":8080";
14+
15+
static Model2Vec createDefault() {
16+
return new Builder().build();
17+
}
18+
19+
static Model2Vec.Builder custom() {
20+
return new Builder();
21+
}
22+
23+
public static class Builder {
24+
private String versionTag;
25+
26+
public Builder() {
27+
this.versionTag = VERSION;
28+
}
29+
30+
public Model2Vec build() {
31+
var container = new Model2Vec(DOCKER_IMAGE + ":" + versionTag);
32+
container.withCreateContainerCmdModifier(cmd -> cmd.withHostName(HOST_NAME));
33+
return container;
34+
}
35+
}
36+
37+
public Model2Vec(String image) {
38+
super(image);
39+
}
40+
}

‎src/it/java/io/weaviate/containers/Weaviate.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,9 @@ public Builder withDefaultVectorizer(String module) {
139139
return this;
140140
}
141141

142-
public Builder withContextionaryUrl(String url) {
143-
addModules(Contextionary.MODULE);
144-
environment.put("CONTEXTIONARY_URL", url);
142+
public Builder withModel2VecUrl(String url) {
143+
addModules(Model2Vec.MODULE);
144+
environment.put("MODEL2VEC_INFERENCE_API", "http://" + url);
145145
return this;
146146
}
147147

@@ -153,7 +153,7 @@ public Builder withImageInference(String url, String module) {
153153

154154
public Builder withOffloadS3(String accessKey, String secretKey) {
155155
addModules("offload-s3");
156-
environment.put("OFFLOAD_S3_ENDPOINT", "http://minio:9000");
156+
environment.put("OFFLOAD_S3_ENDPOINT", "http://" + MinIo.URL);
157157
environment.put("OFFLOAD_S3_BUCKET_AUTO_CREATE", "true");
158158
environment.put("AWS_ACCESS_KEY_ID", accessKey);
159159
environment.put("AWS_SECRET_KEY", secretKey);

‎src/it/java/io/weaviate/integration/SearchITest.java‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -41,19 +41,19 @@
4141
import io.weaviate.client6.v1.api.collections.vectorindex.MultiVector;
4242
import io.weaviate.containers.Container;
4343
import io.weaviate.containers.Container.ContainerGroup;
44-
import io.weaviate.containers.Contextionary;
4544
import io.weaviate.containers.Img2VecNeural;
45+
import io.weaviate.containers.Model2Vec;
4646
import io.weaviate.containers.Weaviate;
4747

4848
public class SearchITest extends ConcurrentTest {
4949
private static final ContainerGroup compose = Container.compose(
5050
Weaviate.custom()
51-
.withContextionaryUrl(Contextionary.URL)
51+
.withModel2VecUrl(Model2Vec.URL)
5252
.withImageInference(Img2VecNeural.URL, Img2VecNeural.MODULE)
5353
.addModules("generative-dummy")
5454
.build(),
5555
Container.IMG2VEC_NEURAL,
56-
Container.CONTEXTIONARY);
56+
Container.MODEL2VEC);
5757
@ClassRule // Bind containers to the lifetime of the test
5858
public static final TestRule _rule = compose.asTestRule();
5959
private static final WeaviateClient client = compose.getClient();
@@ -151,7 +151,7 @@ public void testNearText() throws IOException {
151151
client.collections.create(nsSongs,
152152
col -> col
153153
.properties(Property.text("title"))
154-
.vectorConfig(VectorConfig.text2vecContextionary()));
154+
.vectorConfig(VectorConfig.text2vecModel2Vec()));
155155

156156
var songs = client.collections.use(nsSongs);
157157
var submarine = songs.data.insert(Map.of("title", "Yellow Submarine"));
@@ -160,7 +160,7 @@ public void testNearText() throws IOException {
160160

161161
var result = songs.query.nearText("forest",
162162
opt -> opt
163-
.distance(0.5f)
163+
.distance(0.9f)
164164
.moveTo(.98f, to -> to.concepts("tropical"))
165165
.moveAway(.4f, away -> away.uuids(submarine.metadata().uuid()))
166166
.returnProperties("title"));
@@ -173,7 +173,7 @@ public void testNearText() throws IOException {
173173

174174
@Test
175175
public void testNearText_groupBy() throws IOException {
176-
var vectorizer = VectorConfig.text2vecContextionary();
176+
var vectorizer = VectorConfig.text2vecModel2Vec();
177177

178178
var nsArtists = ns("Artists");
179179
client.collections.create(nsArtists,
@@ -370,7 +370,7 @@ public void testNearObject() throws IOException {
370370
client.collections.create(nsAnimals,
371371
collection -> collection
372372
.properties(Property.text("kind"))
373-
.vectorConfig(VectorConfig.text2vecContextionary()));
373+
.vectorConfig(VectorConfig.text2vecModel2Vec()));
374374

375375
var animals = client.collections.use(nsAnimals);
376376

@@ -399,7 +399,7 @@ public void testHybrid() throws IOException {
399399
client.collections.create(nsHobbies,
400400
collection -> collection
401401
.properties(Property.text("name"), Property.text("description"))
402-
.vectorConfig(VectorConfig.text2vecContextionary()));
402+
.vectorConfig(VectorConfig.text2vecModel2Vec()));
403403

404404
var hobbies = client.collections.use(nsHobbies);
405405

@@ -432,7 +432,7 @@ public void testBadRequest() throws IOException {
432432
client.collections.create(nsThings,
433433
collection -> collection
434434
.properties(Property.text("name"))
435-
.vectorConfig(VectorConfig.text2vecContextionary()));
435+
.vectorConfig(VectorConfig.text2vecModel2Vec()));
436436

437437
var things = client.collections.use(nsThings);
438438
var balloon = things.data.insert(Map.of("name", "balloon"));
@@ -449,7 +449,7 @@ public void testBadRequest_async() throws Throwable {
449449
async.collections.create(nsThings,
450450
collection -> collection
451451
.properties(Property.text("name"))
452-
.vectorConfig(VectorConfig.text2vecContextionary()))
452+
.vectorConfig(VectorConfig.text2vecModel2Vec()))
453453
.join();
454454

455455
var things = async.collections.use(nsThings);
@@ -470,7 +470,7 @@ public void testMetadataAll() throws IOException {
470470
client.collections.create(nsThings,
471471
c -> c
472472
.properties(Property.text("name"))
473-
.vectorConfig(VectorConfig.text2vecContextionary(
473+
.vectorConfig(VectorConfig.text2vecModel2Vec(
474474
t2v -> t2v.sourceProperties("name"))));
475475

476476
var things = client.collections.use(nsThings);
@@ -563,7 +563,7 @@ public void testGenerative_bm25() throws IOException {
563563
c -> c
564564
.properties(Property.text("title"))
565565
.generativeModule(new DummyGenerative())
566-
.vectorConfig(VectorConfig.text2vecContextionary(
566+
.vectorConfig(VectorConfig.text2vecModel2Vec(
567567
t2v -> t2v.sourceProperties("title"))));
568568

569569
var things = client.collections.use(nsThings);
@@ -604,7 +604,7 @@ public void testGenerative_bm25_groupBy() throws IOException {
604604
c -> c
605605
.properties(Property.text("title"))
606606
.generativeModule(new DummyGenerative())
607-
.vectorConfig(VectorConfig.text2vecContextionary(
607+
.vectorConfig(VectorConfig.text2vecModel2Vec(
608608
t2v -> t2v.sourceProperties("title"))));
609609

610610
var things = client.collections.use(nsThings);

‎src/main/java/io/weaviate/client6/v1/api/collections/Generative.java‎

Lines changed: 51 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -94,27 +94,47 @@ public static Generative anyscale(Function<AnyscaleGenerative.Builder, ObjectBui
9494
}
9595

9696
/**
97-
* Configure a default {@code generative-aws} module.
97+
* Configure a default {@code generative-aws} module with Bedrock integration.
98+
*
99+
* @param region AWS region.
100+
* @param model Model to use with Bedrock service.
101+
*/
102+
public static Generative awsBedrock(String region, String model) {
103+
return AwsGenerative.bedrock(region, model);
104+
}
105+
106+
/**
107+
* Configure a {@code generative-aws} module with Bedrock integration.
108+
*
109+
* @param region AWS region.
110+
* @param model Model to use with Bedrock service.
111+
* @param fn Lambda expression for optional parameters.
112+
*/
113+
public static Generative awsBedrock(String region, String model,
114+
Function<AwsGenerative.BedrockBuilder, ObjectBuilder<AwsGenerative>> fn) {
115+
return AwsGenerative.bedrock(region, model, fn);
116+
}
117+
118+
/**
119+
* Configure a default {@code generative-aws} module with Sagemaker integration.
98120
*
99121
* @param region AWS region.
100-
* @param service AWS service to use, e.g. {@code "bedrock"} or
101-
* {@code "sagemaker"}.
122+
* @param baseUrl Base inference URL.
102123
*/
103-
public static Generative aws(String region, String service) {
104-
return AwsGenerative.of(region, service);
124+
public static Generative awsSagemaker(String region, String baseUrl) {
125+
return AwsGenerative.sagemaker(region, baseUrl);
105126
}
106127

107128
/**
108-
* Configure a {@code generative-aws} module.
129+
* Configure a {@code generative-aws} module with Sagemaker integration.
109130
*
110131
* @param region AWS region.
111-
* @param service AWS service to use, e.g. {@code "bedrock"} or
112-
* {@code "sagemaker"}.
132+
* @param baseUrl Base inference URL.
113133
* @param fn Lambda expression for optional parameters.
114134
*/
115-
public static Generative aws(String region, String service,
116-
Function<AwsGenerative.Builder, ObjectBuilder<AwsGenerative>> fn) {
117-
return AwsGenerative.of(region, service, fn);
135+
public static Generative awsSagemaker(String region, String baseUrl,
136+
Function<AwsGenerative.SagemakerBuilder, ObjectBuilder<AwsGenerative>> fn) {
137+
return AwsGenerative.sagemaker(region, baseUrl, fn);
118138
}
119139

120140
/** Configure a default {@code generative-cohere} module. */
@@ -166,8 +186,8 @@ public static Generative friendliai(Function<FriendliaiGenerative.Builder, Objec
166186
}
167187

168188
/** Configure a default {@code generative-palm} module. */
169-
public static Generative google(String projectId) {
170-
return GoogleGenerative.of(projectId);
189+
public static Generative googleVertex(String projectId) {
190+
return GoogleGenerative.vertex(projectId);
171191
}
172192

173193
/**
@@ -176,9 +196,24 @@ public static Generative google(String projectId) {
176196
* @param projectId Project ID.
177197
* @param fn Lambda expression for optional parameters.
178198
*/
179-
public static Generative google(String projectId,
180-
Function<GoogleGenerative.Builder, ObjectBuilder<GoogleGenerative>> fn) {
181-
return GoogleGenerative.of(projectId, fn);
199+
public static Generative googleVertex(String projectId,
200+
Function<GoogleGenerative.VertexBuilder, ObjectBuilder<GoogleGenerative>> fn) {
201+
return GoogleGenerative.vertex(projectId, fn);
202+
}
203+
204+
/** Configure a default {@code generative-palm} module. */
205+
public static Generative googleAiStudio() {
206+
return GoogleGenerative.aiStudio();
207+
}
208+
209+
/**
210+
* Configure a {@code generative-palm} module.
211+
*
212+
* @param fn Lambda expression for optional parameters.
213+
*/
214+
public static Generative googleAiStudio(
215+
Function<GoogleGenerative.AiStudioBuilder, ObjectBuilder<GoogleGenerative>> fn) {
216+
return GoogleGenerative.aiStudio(fn);
182217
}
183218

184219
/** Configure a default {@code generative-mistral} module. */

0 commit comments

Comments
 (0)