add support for OpenSearch vector store: Squashed

pom.xml

@@ -74,6 +74,8 @@
 		<module>vector-stores/spring-ai-elasticsearch-store</module>
 		<module>spring-ai-spring-boot-starters/spring-ai-starter-watsonx-ai</module>
 		<module>spring-ai-spring-boot-starters/spring-ai-starter-elasticsearch-store</module>
+		<module>vector-stores/spring-ai-opensearch-store</module>
+		<module>spring-ai-spring-boot-starters/spring-ai-starter-opensearch-store</module>
 	</modules>
 
 	<organization>
@@ -151,6 +153,12 @@
 		<azure-search.version>11.6.1</azure-search.version>
 		<weaviate-client.version>4.5.1</weaviate-client.version>
 		<qdrant.version>1.7.1</qdrant.version>
+		<opensearch-client.version>2.10.1</opensearch-client.version>
+		<httpclient5.version>5.3.1</httpclient5.version>
+
+		<!-- testing dependencies -->
+		<testcontainers.version>1.19.7</testcontainers.version>
+		<testcontainers.opensearch.version>2.0.1</testcontainers.opensearch.version>
 
 		<!-- documentation dependencies -->
 		<io.spring.maven.antora-version>0.0.4</io.spring.maven.antora-version>

spring-ai-bom/pom.xml

@@ -210,6 +210,12 @@
 				<version>${project.version}</version>
 			</dependency>
 
+            <dependency>
+                <groupId>org.springframework.ai</groupId>
+                <artifactId>spring-ai-opensearch-store</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+
             <!-- Utilities -->
             <dependency>
                 <groupId>org.springframework.ai</groupId>

spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/opensearch.adoc

@@ -0,0 +1,227 @@
+= OpenSearch
+
+This section guides you through setting up the OpenSearch `VectorStore` to store document embeddings and perform similarity searches.
+
+link:https://opensearch.org[OpenSearch] is an open-source search and analytics engine originally forked from Elasticsearch, distributed under the Apache License 2.0. It enhances AI application development by simplifying the integration and management of AI-generated assets. OpenSearch supports vector, lexical, and hybrid search capabilities, leveraging advanced vector database functionalities to facilitate low-latency queries and similarity searches as detailed on the link:https://opensearch.org/platform/search/vector-database.html[vector database page]. This platform is ideal for building scalable AI-driven applications and offers robust tools for data management, fault tolerance, and resource access controls.
+
+== Prerequisites
+
+* A running OpenSearch instance. The following options are available:
+** link:https://opensearch.org/docs/latest/opensearch/install/index/[Self-Managed OpenSearch]
+** link:https://docs.aws.amazon.com/opensearch-service/[Amazon OpenSearch Service]
+* `EmbeddingClient` instance to compute the document embeddings. Several options are available:
+- If required, an API key for the xref:api/embeddings.adoc#available-implementations[EmbeddingClient] to generate the
+embeddings stored by the `OpenSearchVectorStore`.
+
+== Dependencies
+
+Add the OpenSearch Vector Store dependency to your project:
+
+[source,xml]
+----
+<dependency>
+    <groupId>org.springframework.ai</groupId>
+    <artifactId>spring-ai-opensearch-store</artifactId>
+</dependency>
+----
+
+or to your Gradle `build.gradle` build file.
+
+[source,groovy]
+----
+dependencies {
+    implementation 'org.springframework.ai:spring-ai-opensearch-store'
+}
+----
+
+TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file.
+
+== Configuration
+
+To connect to OpenSearch and use the `OpenSearchVectorStore`, you need to provide access details for your instance.
+A simple configuration can either be provided via Spring Boot's `application.yml`,
+[source,yaml]
+----
+spring:
+  opensearch:
+    uris: <opensearch instance URIs>
+    username: <opensearch username>
+    password: <opensearch password>
+    indexName: <opensearch index name>
+    mappingJson: <JSON mapping for opensearch index>
+# API key if needed, e.g. OpenAI
+  ai:
+    openai:
+      api:
+        key: <api-key>
+----
+TIP: Check the list of xref:#_configuration_properties[configuration parameters] to learn about the default values and configuration options.
+
+== Auto-configuration
+
+Spring AI provides Spring Boot auto-configuration for the OpenSearch Vector Store.
+To enable it, add the following dependency to your project's Maven `pom.xml` file:
+
+[source,xml]
+----
+<dependency>
+    <groupId>org.springframework.ai</groupId>
+    <artifactId>spring-ai-opensearch-store-spring-boot-starter</artifactId>
+</dependency>
+----
+
+or to your Gradle `build.gradle` build file.
+
+[source,groovy]
+----
+dependencies {
+    implementation 'org.springframework.ai:spring-ai-opensearch-store-spring-boot-starter'
+}
+----
+
+TIP: Refer to the xref:getting-started.adoc#dependency-management[Dependency Management] section to add the Spring AI BOM to your build file.
+
+Here is an example of the needed bean:
+
+[source,java]
+----
+@Bean
+public EmbeddingClient embeddingClient() {
+    // Can be any other EmbeddingClient implementation
+    return new OpenAiEmbeddingClient(new OpenAiApi(System.getenv("SPRING_AI_OPENAI_API_KEY")));
+}
+----
+
+Now you can auto-wire the `OpenSearchVectorStore` as a vector store in your application.
+
+[source,java]
+----
+@Autowired VectorStore vectorStore;
+
+// ...
+
+List <Document> documents = List.of(
+    new Document("Spring AI rocks!! Spring AI rocks!! Spring AI rocks!! Spring AI rocks!! Spring AI rocks!!", Map.of("meta1", "meta1")),
+    new Document("The World is Big and Salvation Lurks Around the Corner"),
+    new Document("You walk forward facing the past and you turn back toward the future.", Map.of("meta2", "meta2")));
+
+// Add the documents to OpenSearch
+vectorStore.add(List.of(document));
+
+// Retrieve documents similar to a query
+List<Document> results = vectorStore.similaritySearch(SearchRequest.query("Spring").withTopK(5));
+----
+
+=== Configuration properties
+
+You can use the following properties in your Spring Boot configuration to customize the PGVector vector store.
+
+[cols="2,5,1"]
+|===
+|Property| Description | Default value
+
+|`spring.opensearch.uris`| URIs of the OpenSearch cluster endpoints. | -
+|`spring.opensearch.username`| Username for accessing the OpenSearch cluster. | -
+|`spring.opensearch.password`| Password for the specified username. | -
+|`spring.opensearch.indexName`| Name of the default index to be used within the OpenSearch cluster. | `spring-ai-document-index`
+|`spring.opensearch.mappingJson`| JSON string defining the mapping for the index; specifies how documents and their
+fields are stored and indexed. |
+{
+    "properties":{
+        "embedding":{
+        "type":"knn_vector",
+        "dimension":1536
+        }
+    }
+}
+|===
+
+=== Customizing OpenSearch Client Configuration
+
+In cases where the Spring Boot auto-configured OpenSearchClient with `Apache HttpClient 5 Transport` bean is not what
+you want or need, you can still define your own bean.
+Please read the link:https://opensearch.org/docs/latest/clients/java/[OpenSearch Java Client Documentation]
+
+for more in-depth information about the configuration of Amazon OpenSearch Service.
+To enable it, add the following dependency to your project's Maven `pom.xml` file:
+
+[source,xml]
+----
+<dependency>
+    <groupId>software.amazon.awssdk</groupId>
+    <artifactId>apache-client</artifactId>
+    <version>2.25.40</version>
+</dependency>
+----
+
+or to your Gradle `build.gradle` build file.
+
+[source,groovy]
+----
+dependencies {
+    implementation 'software.amazon.awssdk:apache-client:2.25.40'
+}
+----
+
+Here is an example of the needed bean:
+
+[source,java]
+----
+@Bean
+public OpenSearchClient openSearchClient() {
+    return new OpenSearchClient(
+            new AwsSdk2Transport(
+                    ApacheHttpClient.builder().build(),
+                    "search-...us-west-2.es.amazonaws.com", // OpenSearch endpoint, without https://
+                    "es",
+                    Region.US_WEST_2, // signing service region
+                    AwsSdk2TransportOptions.builder().build())
+    );
+}
+----
+
+== Metadata Filtering
+
+You can leverage the generic, portable xref:api/vectordbs.adoc#metadata-filters[metadata filters] with OpenSearch as well.
+
+For example, you can use either the text expression language:
+
+[source,java]
+----
+vectorStore.similaritySearch(SearchRequest.defaults()
+        .withQuery("The World")
+        .withTopK(TOP_K)
+        .withSimilarityThreshold(SIMILARITY_THRESHOLD)
+        .withFilterExpression("author in ['john', 'jill'] && 'article_type' == 'blog'"));
+----
+
+or programmatically using the `Filter.Expression` DSL:
+
+[source,java]
+----
+FilterExpressionBuilder b = new FilterExpressionBuilder();
+
+vectorStore.similaritySearch(SearchRequest.defaults()
+        .withQuery("The World")
+        .withTopK(TOP_K)
+        .withSimilarityThreshold(SIMILARITY_THRESHOLD)
+        .withFilterExpression(b.and(
+                b.in("john", "jill"),
+                b.eq("article_type", "blog")).build()));
+----
+
+NOTE: Those (portable) filter expressions get automatically converted into the proprietary OpenSearch link:https://opensearch.org/docs/latest/query-dsl/full-text/query-string/[Query string query].
+
+For example, this portable filter expression:
+
+[source,sql]
+----
+author in ['john', 'jill'] && 'article_type' == 'blog'
+----
+
+is converted into the proprietary OpenSearch filter format:
+
+[source,text]
+----
+(metadata.author:john OR jill) AND metadata.article_type:blog
+----

spring-ai-spring-boot-autoconfigure/pom.xml

@@ -267,6 +267,13 @@
 			<optional>true</optional>
 		</dependency>
 
+		<dependency>
+			<groupId>org.springframework.ai</groupId>
+			<artifactId>spring-ai-opensearch-store</artifactId>
+			<version>${project.parent.version}</version>
+			<optional>true</optional>
+		</dependency>
+
 		<!-- test dependencies -->
 
 		<dependency>
@@ -354,6 +361,13 @@
 			<scope>test</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>org.opensearch</groupId>
+			<artifactId>opensearch-testcontainers</artifactId>
+			<version>${testcontainers.opensearch.version}</version>
+			<scope>test</scope>
+		</dependency>
+
 		<dependency>
 			<groupId>org.skyscreamer</groupId>
 			<artifactId>jsonassert</artifactId>

spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/opensearch/OpenSearchVectorStoreAutoConfiguration.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2023 - 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.ai.autoconfigure.vectorstore.opensearch;
+
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.core5.http.HttpHost;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
+import org.springframework.ai.embedding.EmbeddingClient;
+import org.springframework.ai.vectorstore.OpenSearchVectorStore;
+import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+
+import java.net.URISyntaxException;
+import java.util.Optional;
+
+@AutoConfiguration
+@ConditionalOnClass({ OpenSearchVectorStore.class, EmbeddingClient.class, OpenSearchClient.class })
+@EnableConfigurationProperties(OpenSearchVectorStoreProperties.class)
+class OpenSearchVectorStoreAutoConfiguration {
+
+	@Bean
+	@ConditionalOnMissingBean
+	OpenSearchVectorStore vectorStore(OpenSearchVectorStoreProperties properties, OpenSearchClient openSearchClient,
+			EmbeddingClient embeddingClient) {
+		return new OpenSearchVectorStore(
+				Optional.ofNullable(properties.getIndexName()).orElse(OpenSearchVectorStore.DEFAULT_INDEX_NAME),
+				openSearchClient, embeddingClient, Optional.ofNullable(properties.getMappingJson())
+					.orElse(OpenSearchVectorStore.DEFAULT_MAPPING_EMBEDDING_TYPE_KNN_VECTOR_DIMENSION_1536));
+	}
+
+	@Bean
+	@ConditionalOnMissingBean
+	OpenSearchClient openSearchClient(OpenSearchVectorStoreProperties properties) {
+		HttpHost[] httpHosts = properties.getUris().stream().map(s -> createHttpHost(s)).toArray(HttpHost[]::new);
+		ApacheHttpClient5TransportBuilder transportBuilder = ApacheHttpClient5TransportBuilder.builder(httpHosts);
+
+		Optional.ofNullable(properties.getUsername())
+			.map(username -> createBasicCredentialsProvider(httpHosts[0], username, properties.getPassword()))
+			.ifPresent(basicCredentialsProvider -> transportBuilder
+				.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder
+					.setDefaultCredentialsProvider(basicCredentialsProvider)));
+
+		return new OpenSearchClient(transportBuilder.build());
+	}
+
+	private BasicCredentialsProvider createBasicCredentialsProvider(HttpHost httpHost, String username,
+			String password) {
+		BasicCredentialsProvider basicCredentialsProvider = new BasicCredentialsProvider();
+		basicCredentialsProvider.setCredentials(new AuthScope(httpHost),
+				new UsernamePasswordCredentials(username, password.toCharArray()));
+		return basicCredentialsProvider;
+	}
+
+	private HttpHost createHttpHost(String s) {
+		try {
+			return HttpHost.create(s);
+		}
+		catch (URISyntaxException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+}