diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/pom.xml b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/pom.xml new file mode 100644 index 00000000000..043670eb9fd --- /dev/null +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/pom.xml @@ -0,0 +1,85 @@ + + + + + 4.0.0 + + org.springframework.ai + spring-ai-parent + 1.1.0-SNAPSHOT + ../../../pom.xml + + spring-ai-autoconfigure-vector-store-s3 + jar + Spring AI Auto Configuration for S3 vector store + Spring AI Auto Configuration for S3 vector store + https://github.com/spring-projects/spring-ai + + + https://github.com/spring-projects/spring-ai + git://github.com/spring-projects/spring-ai.git + git@github.com:spring-projects/spring-ai.git + + + + + + org.springframework.ai + spring-ai-s3-vector-store + ${project.parent.version} + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.springframework.boot + spring-boot-autoconfigure-processor + true + + + + org.springframework.ai + spring-ai-test + ${project.parent.version} + test + + + org.springframework.boot + spring-boot-starter-test + test + + + org.awaitility + awaitility + test + + + org.springframework.ai + spring-ai-transformers + ${project.parent.version} + test + + + diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/main/java/org/springframework/ai/vectorstore/s3/autoconfigure/S3VectorStoreAutoConfiguration.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/main/java/org/springframework/ai/vectorstore/s3/autoconfigure/S3VectorStoreAutoConfiguration.java new file mode 100644 index 00000000000..4ddad7e3645 --- /dev/null +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/main/java/org/springframework/ai/vectorstore/s3/autoconfigure/S3VectorStoreAutoConfiguration.java @@ -0,0 +1,58 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.s3.autoconfigure; + + +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.SpringAIVectorStoreTypes; +import org.springframework.ai.vectorstore.s3.S3VectorStore; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.util.Assert; +import software.amazon.awssdk.services.s3vectors.S3VectorsClient; + +/** + * {@link AutoConfiguration Auto-configuration} for S3 Vector Store. + * + * @author Matej Nedic + */ +@AutoConfiguration +@ConditionalOnClass({S3VectorsClient.class, EmbeddingModel.class}) +@EnableConfigurationProperties(S3VectorStoreProperties.class) +@ConditionalOnProperty(name = SpringAIVectorStoreTypes.TYPE, havingValue = SpringAIVectorStoreTypes.S3, + matchIfMissing = true) +public class S3VectorStoreAutoConfiguration { + + private final S3VectorStoreProperties properties; + + S3VectorStoreAutoConfiguration(S3VectorStoreProperties p) { + Assert.notNull(p.getIndexName(), "Index name cannot be null!"); + Assert.notNull(p.getVectorBucketName(), "Bucket name cannot be null"); + this.properties = p; + } + @Bean + @ConditionalOnMissingBean + S3VectorStore s3VectorStore(S3VectorsClient s3VectorsClient, EmbeddingModel embeddingModel) { + S3VectorStore.Builder builder = new S3VectorStore.Builder(s3VectorsClient, embeddingModel); + builder.indexName(properties.getIndexName()).vectorBucketName(properties.getVectorBucketName()); + return builder.build(); + } +} diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/main/java/org/springframework/ai/vectorstore/s3/autoconfigure/S3VectorStoreProperties.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/main/java/org/springframework/ai/vectorstore/s3/autoconfigure/S3VectorStoreProperties.java new file mode 100644 index 00000000000..dab61a5dc91 --- /dev/null +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/main/java/org/springframework/ai/vectorstore/s3/autoconfigure/S3VectorStoreProperties.java @@ -0,0 +1,50 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.s3.autoconfigure; + + +import org.springframework.boot.context.properties.ConfigurationProperties; + + +/** + * @author Matej Nedic + */ +@ConfigurationProperties(prefix = S3VectorStoreProperties.CONFIG_PREFIX) +public class S3VectorStoreProperties { + + public static final String CONFIG_PREFIX = "spring.ai.vectorstore.s3"; + + private String indexName; + + private String vectorBucketName; + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getVectorBucketName() { + return vectorBucketName; + } + + public void setVectorBucketName(String vectorBucketName) { + this.vectorBucketName = vectorBucketName; + } +} diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000000..15c020683e7 --- /dev/null +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1,16 @@ +# +# Copyright 2025-2025 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.springframework.ai.vectorstore.s3.autoconfigure.S3VectorStoreAutoConfiguration diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/test/java/org/springframework/ai/vectorstore/azure/autoconfigure/S3VectorStoreAutoConfigurationTest.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/test/java/org/springframework/ai/vectorstore/azure/autoconfigure/S3VectorStoreAutoConfigurationTest.java new file mode 100644 index 00000000000..b45c501c1bb --- /dev/null +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3/src/test/java/org/springframework/ai/vectorstore/azure/autoconfigure/S3VectorStoreAutoConfigurationTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.azure.autoconfigure; + + +import io.micrometer.observation.tck.TestObservationRegistry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.transformers.TransformersEmbeddingModel; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.ai.vectorstore.s3.S3VectorStore; +import org.springframework.ai.vectorstore.s3.autoconfigure.S3VectorStoreAutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.boot.test.system.OutputCaptureExtension; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3vectors.S3VectorsClient; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Matej Nedic + */ +@ExtendWith(OutputCaptureExtension.class) +public class S3VectorStoreAutoConfigurationTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of( S3VectorStoreAutoConfiguration.class)) + .withUserConfiguration(Config.class) + .withPropertyValues("spring.ai.vectorstore.s3.vectorBucketName=testBucket") + .withPropertyValues("spring.ai.vectorstore.s3.indexName=testIndex"); + + @Test + public void autoConfigurationDisabledWhenTypeIsNone() { + this.contextRunner.withPropertyValues("spring.ai.vectorstore.type=none").run(context -> { + assertThat(context.getBeansOfType(S3VectorStore.class)).isEmpty(); + assertThat(context.getBeansOfType(VectorStore.class)).isEmpty(); + }); + } + + @Test + public void autoConfigurationEnabledByDefault() { + this.contextRunner.run(context -> { + assertThat(context.getBeansOfType(S3VectorStore.class)).isNotEmpty(); + assertThat(context.getBeansOfType(VectorStore.class)).isNotEmpty(); + assertThat(context.getBean(VectorStore.class)).isInstanceOf(S3VectorStore.class); + }); + } + + @Test + public void autoConfigurationEnabledWhenTypeIsS3() { + this.contextRunner.withPropertyValues("spring.ai.vectorstore.type=S3").run(context -> { + assertThat(context.getBeansOfType(S3VectorStore.class)).isNotEmpty(); + assertThat(context.getBeansOfType(VectorStore.class)).isNotEmpty(); + assertThat(context.getBean(VectorStore.class)).isInstanceOf(S3VectorStore.class); + }); + } + + @Configuration(proxyBeanMethods = false) + static class Config { + + @Bean + public S3VectorsClient s3VectorsClient() { + return S3VectorsClient.builder().region(Region.US_EAST_1).credentialsProvider(DefaultCredentialsProvider.builder().build()).build(); + } + + @Bean + public EmbeddingModel embeddingModel() { + return new TransformersEmbeddingModel(); + } + + } + + +} diff --git a/pom.xml b/pom.xml index 713d2f3335b..3bb7c850bf6 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ vector-stores/spring-ai-redis-store vector-stores/spring-ai-typesense-store vector-stores/spring-ai-weaviate-store - + vector-stores/spring-ai-s3-vector-store auto-configurations/common/spring-ai-autoconfigure-retry @@ -140,6 +140,7 @@ auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-typesense auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-pgvector + auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-s3 spring-ai-spring-boot-starters/spring-ai-starter-vector-store-aws-opensearch spring-ai-spring-boot-starters/spring-ai-starter-vector-store-azure @@ -161,6 +162,7 @@ spring-ai-spring-boot-starters/spring-ai-starter-vector-store-redis spring-ai-spring-boot-starters/spring-ai-starter-vector-store-typesense spring-ai-spring-boot-starters/spring-ai-starter-vector-store-weaviate + spring-ai-spring-boot-starters/spring-ai-starter-vector-store-s3 models/spring-ai-anthropic models/spring-ai-azure-openai diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/VectorStoreProvider.java b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/VectorStoreProvider.java index a9c3e3f2b12..8ed61aba777 100644 --- a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/VectorStoreProvider.java +++ b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/VectorStoreProvider.java @@ -120,6 +120,11 @@ public enum VectorStoreProvider { */ REDIS("redis"), + /** + * Vector store provided by simple. + */ + S3_VECTOR("s3_vector"), + /** * Vector store provided by simple. */ diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc index 8e7ccdd6258..57026c95ff7 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc @@ -107,6 +107,7 @@ *** xref:api/vectordbs/hana.adoc[SAP Hana] *** xref:api/vectordbs/typesense.adoc[] *** xref:api/vectordbs/weaviate.adoc[] +*** xref:api/vectordbs/s3-vector-store.adoc[] ** xref:observability/index.adoc[] diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs.adoc index 10bb7828331..f58fe4c3ff3 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs.adoc @@ -394,6 +394,7 @@ These are the available implementations of the `VectorStore` interface: * xref:api/vectordbs/hana.adoc[SAP Hana Vector Store] - The https://news.sap.com/2024/04/sap-hana-cloud-vector-engine-ai-with-business-context/[SAP HANA] vector store. * xref:api/vectordbs/typesense.adoc[Typesense Vector Store] - The https://typesense.org/docs/0.24.0/api/vector-search.html[Typesense] vector store. * xref:api/vectordbs/weaviate.adoc[Weaviate Vector Store] - The https://weaviate.io/[Weaviate] vector store. +* xref:api/vectordbs/s3-vector-store.adoc[S3 Vector Store] - The https://aws.amazon.com/s3/features/vectors/[AWS S3] vector store. * link:https://github.com/spring-projects/spring-ai/blob/main/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java[SimpleVectorStore] - A simple implementation of persistent vector storage, good for educational purposes. More implementations may be supported in future releases. diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/s3-vector-store.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/s3-vector-store.adoc new file mode 100644 index 00000000000..55dde9f5238 --- /dev/null +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/s3-vector-store.adoc @@ -0,0 +1,188 @@ += S3 Vector Store + +This section walks you through setting up `S3VectorStore` to store document embeddings and perform similarity searches. + +link:https://aws.amazon.com/s3/features/vectors/[AWS S3 Vector Store] is a serverless object storage which supports storing and querying vector at scale. + +link:https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vectors.html[S3 Vector Store API] extends the core features of AWS S3 Bucket and allows you to use S3 as a vector database: + +* Store vectors and the associated metadata within hashes or JSON documents +* Retrieve vectors +* Perform vector searches + +== Prerequisites + +1. A S3 Vector Store Bucket +- https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vectors-buckets-create.html[How to create S3 Vector Bucket] + +2. `EmbeddingModel` instance to compute the document embeddings. Several options are available: +- If required, an API key for the xref:api/embeddings.adoc#available-implementations[EmbeddingModel] to generate the embeddings stored by the `S3VectorStore`. + +== Auto-configuration + +Spring AI provides Spring Boot auto-configuration for the S3 Vector Store. +To enable it, add the following dependency to your project's Maven `pom.xml` file: + +[source, xml] +---- + + org.springframework.ai + spring-ai-starter-vector-store-s3 + +---- + +or to your Gradle `build.gradle` build file. + +[source,groovy] +---- +dependencies { + implementation 'org.springframework.ai:spring-ai-starter-vector-store-s3' +} +---- + +TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file. + +TIP: Refer to the xref:getting-started.adoc#artifact-repositories[Artifact Repositories] section to add Maven Central and/or Snapshot Repositories to your build file. + +Please have a look at the list of <> for the vector store to learn about the default values and configuration options. + +Additionally, you will need a configured `EmbeddingModel` bean. Refer to the xref:api/embeddings.adoc#available-implementations[EmbeddingModel] section for more information. + +Now you can auto-wire the `S3VectorStore` as a vector store in your application. + +[source,java] +---- +@Autowired VectorStore vectorStore; + +// ... + +List documents = List.of( + new Document("Spring AI rocks!! Spring AI rocks!! Spring AI rocks!! Spring AI rocks!! Spring AI rocks!!", Map.of("meta1", "meta1")), + new Document("The World is Big and Salvation Lurks Around the Corner"), + new Document("You walk forward facing the past and you turn back toward the future.", Map.of("meta2", "meta2"))); + +// Add the documents to S3 Vector Store Bucket +vectorStore.add(documents); + +// Retrieve documents similar to a query +List results = this.vectorStore.similaritySearch(SearchRequest.builder().query("Spring").topK(5).build()); +---- + +[[s3-properties]] +=== Configuration Properties + +To connect to AWS S3 Vector Store and use the `S3VectorStore`, you will need to create a `Bean` of `S3VectorsClient` which needs to be supplied with correct Credentials and Region. + +Properties starting with `spring.ai.vectorstore.s3.*` are used to configure the `S3VectorStore`: + +[cols="2,5,1",stripes=even] +|=== +|Property | Description | Default Value + +|`spring.ai.vectorstore.s3.index-name` | The name of the index to store the vectors | `spring-ai-index` +|`spring.ai.vectorstore.s3.vector-bucket-name` | The name of bucket where vectors are located | `my-vector-bucket-on-aws` +|=== + +== Metadata Filtering + +You can leverage the generic, portable xref:api/vectordbs.adoc#metadata-filters[metadata filters] with S3 Vector Store as well. + +For example, you can use either the text expression language: + +[source,java] +---- +vectorStore.similaritySearch(SearchRequest.builder() + .query("The World") + .topK(TOP_K) + .similarityThreshold(SIMILARITY_THRESHOLD) + .filterExpression("country in ['UK', 'NL'] && year >= 2020").build()); +---- + +or programmatically using the `Filter.Expression` DSL: + +[source,java] +---- +FilterExpressionBuilder b = new FilterExpressionBuilder(); + +vectorStore.similaritySearch(SearchRequest.builder() + .query("The World") + .topK(TOP_K) + .similarityThreshold(SIMILARITY_THRESHOLD) + .filterExpression(b.and( + b.in("country", "UK", "NL"), + b.gte("year", 2020)).build()).build()); +---- + +NOTE: Those (portable) filter expressions get automatically converted into link:https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/core/document/Document.html[AWS SDK Java V2 Filter Document object]. + +For example, this portable filter expression: + +[source,sql] +---- +country in ['UK', 'NL'] && year >= 2020 +---- + +is converted into the proprietary S3 Vector Store filter format: + +[source,text] +---- +@country:{UK | NL} @year:[2020 inf] +---- + +== Manual Configuration + +Instead of using the Spring Boot auto-configuration, you can manually configure the S3 Vector Store. For this you need to add the `spring-ai-s3-vector-store` to your project: + +[source,xml] +---- + + org.springframework.ai + spring-ai-s3-vector-store + +---- + +or to your Gradle `build.gradle` build file. + +[source,groovy] +---- +dependencies { + implementation 'org.springframework.ai:spring-ai-s3-vector-store' +} +---- + +Then create the `S3VectorStore` bean using the builder pattern: + +[source,java] +---- +@Bean +VectorStore s3VectorStore(S3VectorsClient s3VectorsClient, EmbeddingModel embeddingModel) { + S3VectorStore.Builder builder = new S3VectorStore.Builder(s3VectorsClient, embeddingModel); // Required a must + builder.indexName(properties.getIndexName()) // Required indexName must be specified + .vectorBucketName(properties.getVectorBucketName()) // Required vectorBucketName must be specified + .filterExpressionConverter(yourConverter); // Optional if you want to override default filterConverter + return builder.build(); + } + +// This can be any EmbeddingModel implementation +@Bean +public EmbeddingModel embeddingModel() { + return new OpenAiEmbeddingModel(new OpenAiApi(System.getenv("OPENAI_API_KEY"))); +} +---- + +== Accessing the Native Client + +The S3 Vector Store implementation provides access to the underlying native S3VectorsClient client: + +[source,java] +---- +S3VectorStore vectorStore = context.getBean(S3VectorStore.class); +Optional nativeClient = vectorStore.getNativeClient(); + +if (nativeClient.isPresent()) { + S3VectorsClient s3Client = nativeClient.get(); + // Use the native client for S3-Vector-Store-specific operations +} +---- + +The native client gives you access to S3-Vector-Store-specific features and operations that might not be exposed through the `VectorStore` interface. diff --git a/spring-ai-spring-boot-starters/spring-ai-starter-vector-store-s3/pom.xml b/spring-ai-spring-boot-starters/spring-ai-starter-vector-store-s3/pom.xml new file mode 100644 index 00000000000..4d9b597ef9a --- /dev/null +++ b/spring-ai-spring-boot-starters/spring-ai-starter-vector-store-s3/pom.xml @@ -0,0 +1,46 @@ + + 4.0.0 + + org.springframework.ai + spring-ai-parent + 1.1.0-SNAPSHOT + ../../pom.xml + + spring-ai-starter-vector-store-s3 + jar + Spring AI Starter - S3 Vector Store + Spring AI S3 Vector Store Starter + https://github.com/spring-projects/spring-ai + + + https://github.com/spring-projects/spring-ai + git://github.com/spring-projects/spring-ai.git + git@github.com:spring-projects/spring-ai.git + + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.ai + spring-ai-autoconfigure-vector-store-s3 + ${project.parent.version} + + + org.springframework.ai + spring-ai-autoconfigure-vector-store-observation + ${project.parent.version} + + + org.springframework.ai + spring-ai-s3-vector-store + ${project.parent.version} + + + + + + diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SpringAIVectorStoreTypes.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SpringAIVectorStoreTypes.java index 4e8f8913b7a..14ca00e1e37 100644 --- a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SpringAIVectorStoreTypes.java +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/SpringAIVectorStoreTypes.java @@ -64,4 +64,6 @@ private SpringAIVectorStoreTypes() { public static final String WEAVIATE = "weaviate"; + public static final String S3 = "S3"; + } diff --git a/vector-stores/spring-ai-s3-vector-store/pom.xml b/vector-stores/spring-ai-s3-vector-store/pom.xml new file mode 100644 index 00000000000..53f3fba4ddc --- /dev/null +++ b/vector-stores/spring-ai-s3-vector-store/pom.xml @@ -0,0 +1,47 @@ + + 4.0.0 + + org.springframework.ai + spring-ai-parent + 1.1.0-SNAPSHOT + ../../pom.xml + + + spring-ai-s3-vector-store + + + org.springframework.ai + spring-ai-vector-store + ${project.parent.version} + + + + software.amazon.awssdk + s3vectors + 2.32.2 + + + + + + org.springframework.ai + spring-ai-transformers + ${project.parent.version} + test + + + + org.springframework.ai + spring-ai-test + ${project.parent.version} + test + + + + org.springframework.boot + spring-boot-starter-test + test + + + diff --git a/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/DocumentUtils.java b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/DocumentUtils.java new file mode 100644 index 00000000000..3a730e52470 --- /dev/null +++ b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/DocumentUtils.java @@ -0,0 +1,106 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.ai.vectorstore.s3; + +import software.amazon.awssdk.core.document.Document; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.*; + + +/** + * Helper class to convert from AWS SDK Document to Object and vice versa. + * + * @author Matej Nedic + */ +public class DocumentUtils { + + public static Document toDocument(Object obj) { + if (obj == null) { + return Document.fromNull(); + } else if (obj instanceof String) { + return Document.fromString((String) obj); + } else if (obj instanceof Integer) { + return Document.fromNumber((Integer) obj); + } else if (obj instanceof Long) { + return Document.fromNumber((Long) obj); + } else if (obj instanceof Double) { + return Document.fromNumber((Double) obj); + } else if (obj instanceof Float) { + return Document.fromNumber((Float) obj); + } else if (obj instanceof Short) { + return Document.fromNumber((Short) obj); + } else if (obj instanceof Byte) { + return Document.fromNumber((Byte) obj); + } else if (obj instanceof BigDecimal) { + return Document.fromNumber((BigDecimal) obj); + } else if (obj instanceof BigInteger) { + return Document.fromNumber((BigInteger) obj); + } else if (obj instanceof Boolean) { + return Document.fromBoolean((Boolean) obj); + } else if (obj instanceof Map map) { + Document.MapBuilder mapBuilder = Document.mapBuilder(); + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey().toString(); + Document valueDoc = toDocument(entry.getValue()); + mapBuilder.putDocument(key, valueDoc); + } + return mapBuilder.build(); + } else { + Collection collection = (Collection) obj; + Document.ListBuilder listDoc = Document.listBuilder(); + for (Object item : collection) { + listDoc.addDocument(toDocument(item)); + } + return listDoc.build(); + } + } + + public static Map fromDocument (Document document) { + if (document.isNull()) { + return null; + } + Map mapDocs = document.asMap(); + Map mapMetadata = new HashMap<>(mapDocs.size()); + for (Map.Entry entry : mapDocs.entrySet()) { + mapMetadata.put(entry.getKey(), fromDocumentToObject(entry.getValue())); + } + return mapMetadata; + } + + private static Object fromDocumentToObject(Document document) { + if (document.isNull()) { + return null; + } else if (document.isString()) { + return document.asString(); + } else if (document.isNumber()) { + // This is same problem DynamoDB sdk has. I am in favour of returning BigDecimal because of floats. + return document.asNumber().bigDecimalValue(); + } else if (document.isBoolean()) { + return document.asBoolean(); + } else if (document.isList()) { + List docs = document.asList(); + List listMetadata = new ArrayList<>(docs.size()); + for (Document item : docs) { + listMetadata.add(fromDocument(item)); + } + return listMetadata; + } else { + return fromDocument(document); + } + } +} diff --git a/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/S3VectorFilterExpressionConverter.java b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/S3VectorFilterExpressionConverter.java new file mode 100644 index 00000000000..2fd50d40ec1 --- /dev/null +++ b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/S3VectorFilterExpressionConverter.java @@ -0,0 +1,34 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.ai.vectorstore.s3; + +import org.springframework.ai.vectorstore.filter.Filter; +import software.amazon.awssdk.core.document.Document; + + +/** + * FilterExpression DLS converter specific for AWS S3 Vector Store since SDK required AWS Document object. + * @author Matej Nedic + */ +public interface S3VectorFilterExpressionConverter { + + /** + * Convert the given {@link Filter.Expression} into a {@link Document} representation. + * @param expression the expression to convert + * @return the converted expression + */ + Document convertExpression(Filter.Expression expression); +} diff --git a/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/S3VectorFilterSearchExpressionConverter.java b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/S3VectorFilterSearchExpressionConverter.java new file mode 100644 index 00000000000..400a90f738e --- /dev/null +++ b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/S3VectorFilterSearchExpressionConverter.java @@ -0,0 +1,151 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.ai.vectorstore.s3; + +import org.springframework.ai.vectorstore.filter.Filter; +import software.amazon.awssdk.core.SdkNumber; +import software.amazon.awssdk.core.document.Document; + +import java.math.BigDecimal; +import java.text.SimpleDateFormat; +import java.util.*; + +/** + * Default implementation of {@link S3VectorFilterExpressionConverter} + * @author Matej Nedic + */ +public class S3VectorFilterSearchExpressionConverter implements S3VectorFilterExpressionConverter { + private final SimpleDateFormat dateFormat; + + public S3VectorFilterSearchExpressionConverter() { + this.dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + private String getOperationSymbol(Filter.ExpressionType exp) { + return switch (exp) { + case AND -> "$and"; + case NOT -> "$not"; + case OR -> "$or"; + case EQ -> "$eq"; + case NE -> "$ne"; + case LT -> "$lt"; + case LTE -> "$lte"; + case GT -> "$gt"; + case GTE -> "$gte"; + case NIN -> "$nin"; + case IN -> "$in"; + default -> throw new UnsupportedOperationException("Not supported expression type: " + exp); + }; + } + + + @Override + public Document convertExpression(Filter.Expression expression) { + String operationType = getOperationSymbol(expression.type()); + switch (expression.type()) { + case EQ: + case NE: + case GTE: + case GT: + case LTE: + case LT: + return Document.fromMap(Map.of( + ((Filter.Key) expression.left()).key(), + Document.fromMap(Map.of(operationType, + wrapValue(expression.right()))) + )); + + case IN: + case NIN: + Document document = wrapValue(expression.right()); + return Document.fromMap(Map.of( + ((Filter.Key) expression.left()).key(), + Document.fromMap(Map.of(operationType, document)) + )); + + case AND: + case OR: + Document leftDocument = wrapValue(expression.left()); + Document rightDocument = wrapValue(expression.right()); + return Document.fromMap(Map.of(operationType, Document.fromList(List.of(leftDocument, rightDocument)))); + + default: + throw new UnsupportedOperationException("Unsupported operator: " + expression.type()); + } + } + + + private Document wrapValue(Filter.Operand operand) { + if (operand instanceof Filter.Value) { + return convertToDocument(((Filter.Value) operand).value()); + } else if (operand instanceof Filter.Key) { + return Document.fromString(((Filter.Key) operand).key()); + } else if (operand instanceof Filter.Group) { + Filter.Expression expression = ((Filter.Group) operand).content(); + return convertExpression(expression); + } else { + return convertExpression((Filter.Expression) operand); + } + } + + private Document convertToDocument(Object value) { + if (value instanceof String s) { + return Document.fromString(s); + } + if (value instanceof Boolean b) { + return Document.fromBoolean(b); + } + if (value instanceof Number n) { + return Document.fromNumber(toSdkNumber(n)); + } + if (value instanceof Date d) { + return Document.fromString(dateFormat.format(d)); + } + if (value instanceof List list) { + List converted = list.stream() + .map(this::convertToDocument) + .toList(); + return Document.fromList(converted); + } + return Document.fromString(String.valueOf(value)); + } + + private SdkNumber toSdkNumber(Number num) { + if (num instanceof BigDecimal bd) { + return SdkNumber.fromBigDecimal(bd); + } + if (num instanceof Integer i) { + return SdkNumber.fromInteger(i); + } + if (num instanceof Long l) { + return SdkNumber.fromLong(l); + } + if (num instanceof Double d) { + return SdkNumber.fromDouble(d); + } + if (num instanceof Float f) { + return SdkNumber.fromFloat(f); + } + if (num instanceof Short s) { + return SdkNumber.fromShort(s); + } + if (num instanceof Byte b) { + return SdkNumber.fromInteger(b.intValue()); + } + throw new IllegalArgumentException("Unsupported Number type: " + num.getClass()); + } +} diff --git a/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/S3VectorStore.java b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/S3VectorStore.java new file mode 100644 index 00000000000..5b5bd3e40c9 --- /dev/null +++ b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/S3VectorStore.java @@ -0,0 +1,210 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.ai.vectorstore.s3; + +import org.springframework.ai.document.Document; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.embedding.EmbeddingOptionsBuilder; +import org.springframework.ai.observation.conventions.VectorStoreProvider; +import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.filter.Filter; +import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore; +import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.util.Assert; +import software.amazon.awssdk.services.s3vectors.S3VectorsClient; +import software.amazon.awssdk.services.s3vectors.model.*; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * @author Matej Nedic + */ +public class S3VectorStore extends AbstractObservationVectorStore implements InitializingBean { + + private final S3VectorsClient s3VectorsClient; + + private final String vectorBucketName; + + private final String indexName; + + private final S3VectorFilterExpressionConverter filterExpressionConverter; + + + /** + * Creates a new S3VectorStore instance with the specified builder + * settings. Initializes observation-related components and the embedding model. + * + * @param builder the builder containing configuration settings + */ + protected S3VectorStore(Builder builder) { + super(builder); + + Assert.notNull(builder.vectorBucketName, "vectorBucketName must not be null"); + Assert.notNull(builder.indexName, "indexName must not be null"); + Assert.notNull(builder.s3VectorsClient, "S3VectorsClient must not be null"); + + this.s3VectorsClient = builder.s3VectorsClient; + this.indexName = builder.indexName; + this.filterExpressionConverter = builder.filterExpressionConverter; + this.vectorBucketName = builder.vectorBucketName; + } + + @Override + public void doAdd(List documents) { + List embedding = this.embeddingModel.embed(documents, EmbeddingOptionsBuilder.builder().build(), + this.batchingStrategy); + + PutVectorsRequest.Builder requestBuilder = PutVectorsRequest.builder(); + requestBuilder.indexName(indexName).vectorBucketName(vectorBucketName); + + List vectors = new ArrayList<>(documents.size()); + for (Document document : documents) { + float[] embs = embedding.get(documents.indexOf(document)); + VectorData vectorData = constructVectorData(embs); + vectors.add(PutInputVector.builder().data(vectorData).key(document.getId()).metadata(constructMetadata(document.getMetadata())).build()); + } + requestBuilder.vectors(vectors); + s3VectorsClient.putVectors(requestBuilder.build()); + } + + @Override + public void doDelete(List idList) { + s3VectorsClient.deleteVectors(DeleteVectorsRequest.builder().keys(idList).indexName(indexName).vectorBucketName(vectorBucketName).build()); + } + + @Override + public void doDelete(Filter.Expression filterExpression) { + Assert.notNull(filterExpression, "Filter expression mus not be null"); + + software.amazon.awssdk.core.document.Document filterDoc = this.filterExpressionConverter.convertExpression(filterExpression); + QueryVectorsRequest request = QueryVectorsRequest.builder().filter(filterDoc).vectorBucketName(vectorBucketName).indexName(indexName).build(); + List keys = s3VectorsClient.queryVectors(request).vectors().stream().map(QueryOutputVector::key).collect(Collectors.toList()); + + s3VectorsClient.deleteVectors(DeleteVectorsRequest.builder().vectorBucketName(this.vectorBucketName).keys(keys).indexName(indexName).build()); + } + + @Override + public List doSimilaritySearch(SearchRequest searchRequest) { + Assert.notNull(searchRequest, "The search request must not be null."); + + QueryVectorsRequest.Builder requestBuilder = + QueryVectorsRequest.builder().indexName(indexName).vectorBucketName(vectorBucketName) + .topK(searchRequest.getTopK()).returnMetadata(true).returnDistance(true); + + if (searchRequest.hasFilterExpression()) { + software.amazon.awssdk.core.document.Document filter = filterExpressionConverter.convertExpression(searchRequest.getFilterExpression()); + requestBuilder.filter(filter); + } + + float[] embeddings = this.embeddingModel.embed(searchRequest.getQuery()); + VectorData vectorData = constructVectorData(embeddings); + requestBuilder.queryVector(vectorData); + + QueryVectorsResponse response = s3VectorsClient.queryVectors(requestBuilder.build()); + return response.vectors().stream().map(this::toDocument).collect(Collectors.toList()); + } + + private Document toDocument(QueryOutputVector vector) { + Map metadata = DocumentUtils.fromDocument(vector.metadata()); + if (vector.distance() != null) { + metadata.put("SPRING_AI_S3_DISTANCE", vector.distance()); + } + return Document.builder().metadata(metadata) + .text(vector.key()).build(); + } + + private static software.amazon.awssdk.core.document.Document constructMetadata(Map originalMetadata) { + Map metadata = new HashMap<>(originalMetadata.size()); + originalMetadata.forEach((k, v) -> + metadata.put(k, DocumentUtils.toDocument(v)) + ); + return software.amazon.awssdk.core.document.Document.fromMap(metadata); + } + + private static VectorData constructVectorData(float[] embedding) { + ArrayList float32 = new ArrayList<>(embedding.length); + for (float v : embedding) { + float32.add(v); + } + return VectorData.builder().float32(float32).build(); + } + + @Override + public VectorStoreObservationContext.Builder createObservationContextBuilder(String operationName) { + return VectorStoreObservationContext.builder(VectorStoreProvider.S3_VECTOR.value(), operationName) + .collectionName(this.indexName) + .dimensions(this.embeddingModel.dimensions()); + } + + @Override + public void afterPropertiesSet() throws Exception { + // Index requires distance and other stuff to be created. Not sure if this is place to do. + // I can provide rather Util Class like builder which creates index. + } + + @Override + public Optional getNativeClient() { + @SuppressWarnings("unchecked") + T client = (T) this.s3VectorsClient; + return Optional.of(client); + } + + + + + public static class Builder extends AbstractVectorStoreBuilder { + private final S3VectorsClient s3VectorsClient; + + private String vectorBucketName; + + private String indexName; + + private S3VectorFilterExpressionConverter filterExpressionConverter = new S3VectorFilterSearchExpressionConverter(); + + public Builder(S3VectorsClient s3VectorsClient, EmbeddingModel embeddingModel) { + super(embeddingModel); + Assert.notNull(s3VectorsClient, "S3VectorsClient must not be null"); + this.s3VectorsClient = s3VectorsClient; + } + + public Builder vectorBucketName(String vectorBucketName) { + Assert.notNull(vectorBucketName, "vectorBucketName must not be null"); + this.vectorBucketName = vectorBucketName; + return this; + } + + public Builder indexName(String indexName) { + Assert.notNull(indexName, "indexName must not be null"); + this.indexName = indexName; + return this; + + } + + public Builder filterExpressionConverter(S3VectorFilterExpressionConverter converter) { + Assert.notNull(converter, "s3VectorFilterExpressionConverter must not be null"); + this.filterExpressionConverter = converter; + return this; + } + + @Override + public S3VectorStore build() { + return new S3VectorStore(this); + } + } +} diff --git a/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/package-info.java b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/package-info.java new file mode 100644 index 00000000000..727bd1fc830 --- /dev/null +++ b/vector-stores/spring-ai-s3-vector-store/src/main/java/org/springframework/ai/vectorstore/s3/package-info.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides the API for embedding observations. + */ +@NonNullApi +@NonNullFields +package org.springframework.ai.vectorstore.s3; + +import org.springframework.lang.NonNullApi; +import org.springframework.lang.NonNullFields; diff --git a/vector-stores/spring-ai-s3-vector-store/src/test/java/S3FilterExpressionConverterTests.java b/vector-stores/spring-ai-s3-vector-store/src/test/java/S3FilterExpressionConverterTests.java new file mode 100644 index 00000000000..290dbc02236 --- /dev/null +++ b/vector-stores/spring-ai-s3-vector-store/src/test/java/S3FilterExpressionConverterTests.java @@ -0,0 +1,273 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.junit.jupiter.api.Test; +import org.springframework.ai.vectorstore.filter.Filter; +import org.springframework.ai.vectorstore.s3.S3VectorFilterSearchExpressionConverter; +import software.amazon.awssdk.core.document.Document; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.*; + +/** + * @author Matej Nedic + */ +class S3FilterExpressionConverterTests { + + private final S3VectorFilterSearchExpressionConverter converter = new S3VectorFilterSearchExpressionConverter(); + + @Test + public void testDate() { + Document vectorExpr = this.converter.convertExpression(new Filter.Expression(EQ, new Filter.Key("activationDate"), + new Filter.Value(new Date(1704637752148L)))); + Document filter = Document.fromMap(Map.of( + "activationDate", Document.fromMap(Map.of( + "$eq", Document.fromString("2024-01-07T14:29:12Z") + )) + )); + + assertThat(vectorExpr).isEqualTo(filter); + + vectorExpr = this.converter.convertExpression( + new Filter.Expression(EQ, new Filter.Key("activationDate"), new Filter.Value("1970-01-01T00:00:02Z"))); + + filter = Document.fromMap(Map.of( + "activationDate", Document.fromMap(Map.of( + "$eq", Document.fromString("1970-01-01T00:00:02Z") + )) + )); + assertThat(vectorExpr).isEqualTo(filter); + } + + @Test + public void testEQ() { + Document vectorExpr = this.converter + .convertExpression(new Filter.Expression(EQ, new Filter.Key("country"), new Filter.Value("BG"))); + + Document filter = Document.fromMap(Map.of( + "country", Document.fromMap(Map.of( + "$eq", Document.fromString("BG") + )) + )); + + assertThat(vectorExpr).isEqualTo(filter); + } + + @Test + public void tesEqAndGte() { + Document vectorExpr = this.converter.convertExpression(new Filter.Expression(AND, + new Filter.Expression(EQ, new Filter.Key("genre"), new Filter.Value("drama")), + new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020)))); + + Document filter = Document.fromMap(Map.of( + "$and", Document.fromList(List.of( + Document.fromMap(Map.of( + "genre", Document.fromMap(Map.of( + "$eq", Document.fromString("drama") + )) + )), + Document.fromMap(Map.of( + "year", Document.fromMap(Map.of( + "$gte", Document.fromNumber(2020) + )) + )) + )) + )); + assertThat(vectorExpr).isEqualTo(filter); + } + + + @Test + public void tesIn() { + List genres = List.of("comedy", "documentary", "drama"); + Document vectorExpr = this.converter.convertExpression(new Filter.Expression(IN, new Filter.Key("genre"), + new Filter.Value(genres))); + + + Document filter = Document.fromMap(Map.of( + "genre", Document.fromMap(Map.of( + "$in", Document.fromList( + genres.stream() + .map(Document::fromString) + .toList() + ) + )) + )); + assertThat(vectorExpr).isEqualTo(filter); + } + + + @Test + public void testNe() { + Document vectorExpr = this.converter.convertExpression( + new Filter.Expression(OR, new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020)), + new Filter.Expression(AND, + new Filter.Expression(EQ, new Filter.Key("country"), new Filter.Value("BG")), + new Filter.Expression(NE, new Filter.Key("city"), new Filter.Value("Sofia"))))); + + Document filter = Document.fromMap(Map.of( + "$or", Document.fromList(List.of( + Document.fromMap(Map.of( + "year", Document.fromMap(Map.of( + "$gte", Document.fromNumber(2020) + )) + )), + Document.fromMap(Map.of( + "$and", Document.fromList(List.of( + Document.fromMap(Map.of( + "country", Document.fromMap(Map.of( + "$eq", Document.fromString("BG") + )) + )), + Document.fromMap(Map.of( + "city", Document.fromMap(Map.of( + "$ne", Document.fromString("Sofia") + )) + )) + )) + )) + )) + )); + + assertThat(vectorExpr).isEqualTo(filter); + } + + @Test + public void testGroup() { + Document vectorExpr = this.converter.convertExpression(new Filter.Expression(AND, + new Filter.Group(new Filter.Expression(OR, + new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020)), + new Filter.Expression(EQ, new Filter.Key("country"), new Filter.Value("BG")))), + new Filter.Expression(NIN, new Filter.Key("city"), new Filter.Value(List.of("Sofia", "Plovdiv"))))); + + Document filter = Document.fromMap(Map.of( + "$and", Document.fromList(List.of( + Document.fromMap(Map.of( + "$or", Document.fromList(List.of( + Document.fromMap(Map.of( + "year", Document.fromMap(Map.of( + "$gte", Document.fromNumber(2020) + )) + )), + Document.fromMap(Map.of( + "country", Document.fromMap(Map.of( + "$eq", Document.fromString("BG") + )) + )) + )) + )), + Document.fromMap(Map.of( + "city", Document.fromMap(Map.of( + "$nin", Document.fromList(List.of( + Document.fromString("Sofia"), + Document.fromString("Plovdiv") + )) + )) + )) + )) + )); + assertThat(vectorExpr) + .isEqualTo(filter); + } + + @Test + public void tesBoolean() { + Document vectorExpr = this.converter.convertExpression(new Filter.Expression(AND, + new Filter.Expression(AND, new Filter.Expression(EQ, new Filter.Key("isOpen"), new Filter.Value(true)), + new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020))), + new Filter.Expression(IN, new Filter.Key("country"), new Filter.Value(List.of("BG", "NL", "US"))))); + + Document filter = Document.fromMap(Map.of( + "$and", Document.fromList(List.of( + Document.fromMap(Map.of( + "$and", Document.fromList(List.of( + Document.fromMap(Map.of( + "isOpen", Document.fromMap(Map.of( + "$eq", Document.fromBoolean(true) + )) + )), + Document.fromMap(Map.of( + "year", Document.fromMap(Map.of( + "$gte", Document.fromNumber(2020) + )) + )) + )) + )), + Document.fromMap(Map.of( + "country", Document.fromMap(Map.of( + "$in", Document.fromList(List.of( + Document.fromString("BG"), + Document.fromString("NL"), + Document.fromString("US") + )) + )) + )) + )) + )); + assertThat(vectorExpr) + .isEqualTo(filter); + } + + @Test + public void testDecimal() { + Document vectorExpr = this.converter.convertExpression(new Filter.Expression(AND, + new Filter.Expression(GTE, new Filter.Key("temperature"), new Filter.Value(-15.6)), + new Filter.Expression(LTE, new Filter.Key("temperature"), new Filter.Value(20.13)))); + + Document filter = Document.fromMap(Map.of( + "$and", Document.fromList(List.of( + Document.fromMap(Map.of( + "temperature", Document.fromMap(Map.of( + "$gte", Document.fromNumber(-15.6) + )) + )), + Document.fromMap(Map.of( + "temperature", Document.fromMap(Map.of( + "$lte", Document.fromNumber(20.13) + )) + )) + )) + )); + + assertThat(vectorExpr).isEqualTo(filter); + } + + @Test + public void testComplexIdentifiers() { + Document vectorExpr = this.converter + .convertExpression(new Filter.Expression(EQ, new Filter.Key("\"country 1 2 3\""), new Filter.Value("BG"))); + Document filter = Document.fromMap(Map.of( + "\"country 1 2 3\"", Document.fromMap(Map.of( + "$eq", Document.fromString("BG") + )) + )); + + assertThat(vectorExpr).isEqualTo(filter); + + vectorExpr = this.converter + .convertExpression(new Filter.Expression(EQ, new Filter.Key("'country 1 2 3'"), new Filter.Value("BG"))); + filter = Document.fromMap(Map.of( + "'country 1 2 3'", Document.fromMap(Map.of( + "$eq", Document.fromString("BG") + )) + )); + assertThat(vectorExpr).isEqualTo(filter); + } + +}