diff --git a/vector-stores/spring-ai-neo4j-store/src/main/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStore.java b/vector-stores/spring-ai-neo4j-store/src/main/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStore.java index 269d85346b3..01a068d9c01 100644 --- a/vector-stores/spring-ai-neo4j-store/src/main/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStore.java +++ b/vector-stores/spring-ai-neo4j-store/src/main/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStore.java @@ -25,17 +25,18 @@ import org.neo4j.driver.Driver; import org.neo4j.driver.SessionConfig; import org.neo4j.driver.Values; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; import org.springframework.ai.document.DocumentMetadata; -import org.springframework.ai.embedding.BatchingStrategy; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.embedding.EmbeddingOptionsBuilder; -import org.springframework.ai.embedding.TokenCountBatchingStrategy; import org.springframework.ai.observation.conventions.VectorStoreProvider; import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric; import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder; import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.ai.vectorstore.filter.Filter; import org.springframework.ai.vectorstore.neo4j.filter.Neo4jVectorFilterExpressionConverter; import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore; import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext; @@ -133,6 +134,8 @@ */ public class Neo4jVectorStore extends AbstractObservationVectorStore implements InitializingBean { + private static final Logger logger = LoggerFactory.getLogger(Neo4jVectorStore.class); + public static final int DEFAULT_EMBEDDING_DIMENSION = 1536; public static final int DEFAULT_TRANSACTION_SIZE = 10_000; @@ -235,6 +238,29 @@ public Optional doDelete(List idList) { } } + @Override + protected void doDelete(Filter.Expression filterExpression) { + Assert.notNull(filterExpression, "Filter expression must not be null"); + + try (var session = this.driver.session(this.sessionConfig)) { + String whereClause = this.filterExpressionConverter.convertExpression(filterExpression); + + // Create Cypher query with transaction batching + String cypher = """ + MATCH (node:%s) WHERE %s + CALL { WITH node DETACH DELETE node } IN TRANSACTIONS OF $transactionSize ROWS + """.formatted(this.label, whereClause); + + var summary = session.run(cypher, Map.of("transactionSize", DEFAULT_TRANSACTION_SIZE)).consume(); + + logger.debug("Deleted {} nodes matching filter expression", summary.counters().nodesDeleted()); + } + catch (Exception e) { + logger.error("Failed to delete nodes by filter: {}", e.getMessage(), e); + throw new IllegalStateException("Failed to delete nodes by filter", e); + } + } + @Override public List doSimilaritySearch(SearchRequest request) { Assert.isTrue(request.getTopK() > 0, "The number of documents to returned must be greater than zero"); diff --git a/vector-stores/spring-ai-neo4j-store/src/test/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStoreIT.java b/vector-stores/spring-ai-neo4j-store/src/test/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStoreIT.java index 5260b34f930..3d74e561920 100644 --- a/vector-stores/spring-ai-neo4j-store/src/test/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStoreIT.java +++ b/vector-stores/spring-ai-neo4j-store/src/test/java/org/springframework/ai/vectorstore/neo4j/Neo4jVectorStoreIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; @@ -39,6 +40,7 @@ import org.springframework.ai.openai.api.OpenAiApi; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.ai.vectorstore.filter.Filter; import org.springframework.ai.vectorstore.filter.FilterExpressionTextParser; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -53,6 +55,7 @@ * @author Michael Simons * @author Christian Tzolov * @author Thomas Vitale + * @author Soby Chacko */ @Testcontainers @EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+") @@ -301,6 +304,99 @@ void ensureIdIndexGetsCreated() { .isTrue()); } + @Test + void deleteByFilter() { + this.contextRunner.run(context -> { + VectorStore vectorStore = context.getBean(VectorStore.class); + + var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2020)); + var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "NL", "year", 2021)); + var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2023)); + + vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2)); + + SearchRequest searchRequest = SearchRequest.builder() + .query("The World") + .topK(5) + .similarityThresholdAll() + .build(); + + List results = vectorStore.similaritySearch(searchRequest); + assertThat(results).hasSize(3); + + Filter.Expression filterExpression = new Filter.Expression(Filter.ExpressionType.EQ, + new Filter.Key("country"), new Filter.Value("BG")); + + vectorStore.delete(filterExpression); + + results = vectorStore.similaritySearch(searchRequest); + assertThat(results).hasSize(1); + assertThat(results.get(0).getMetadata()).containsEntry("country", "NL"); + }); + } + + @Test + void deleteWithStringFilterExpression() { + this.contextRunner.run(context -> { + VectorStore vectorStore = context.getBean(VectorStore.class); + + var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2020)); + var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "NL", "year", 2021)); + var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2023)); + + vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2)); + + var searchRequest = SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build(); + + List results = vectorStore.similaritySearch(searchRequest); + assertThat(results).hasSize(3); + + vectorStore.delete("country == 'BG'"); + + results = vectorStore.similaritySearch(searchRequest); + assertThat(results).hasSize(1); + assertThat(results.get(0).getMetadata()).containsEntry("country", "NL"); + }); + } + + @Test + void deleteWithComplexFilterExpression() { + this.contextRunner.run(context -> { + VectorStore vectorStore = context.getBean(VectorStore.class); + + var doc1 = new Document("Content 1", Map.of("type", "A", "priority", 1L)); + var doc2 = new Document("Content 2", Map.of("type", "A", "priority", 2L)); + var doc3 = new Document("Content 3", Map.of("type", "B", "priority", 1L)); + + vectorStore.add(List.of(doc1, doc2, doc3)); + + // Complex filter expression: (type == 'A' AND priority > 1) + Filter.Expression priorityFilter = new Filter.Expression(Filter.ExpressionType.GT, + new Filter.Key("priority"), new Filter.Value(1)); + Filter.Expression typeFilter = new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("type"), + new Filter.Value("A")); + Filter.Expression complexFilter = new Filter.Expression(Filter.ExpressionType.AND, typeFilter, + priorityFilter); + + vectorStore.delete(complexFilter); + + var results = vectorStore + .similaritySearch(SearchRequest.builder().query("Content").topK(5).similarityThresholdAll().build()); + + assertThat(results).hasSize(2); + assertThat(results.stream().map(doc -> doc.getMetadata().get("type")).collect(Collectors.toList())) + .containsExactlyInAnyOrder("A", "B"); + assertThat(results.stream().map(doc -> doc.getMetadata().get("priority")).collect(Collectors.toList())) + .containsExactlyInAnyOrder(1L, 1L); + }); + } + @SpringBootConfiguration @EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class }) public static class TestApplication { diff --git a/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStore.java b/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStore.java index 0af4515257e..3e46f665787 100644 --- a/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStore.java +++ b/vector-stores/spring-ai-opensearch-store/src/main/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStore.java @@ -32,17 +32,19 @@ import org.opensearch.client.opensearch._types.query_dsl.Query; import org.opensearch.client.opensearch.core.BulkRequest; import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.DeleteByQueryRequest; +import org.opensearch.client.opensearch.core.DeleteByQueryResponse; import org.opensearch.client.opensearch.core.search.Hit; import org.opensearch.client.opensearch.indices.CreateIndexRequest; import org.opensearch.client.opensearch.indices.CreateIndexResponse; import org.opensearch.client.transport.endpoints.BooleanResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; import org.springframework.ai.document.DocumentMetadata; -import org.springframework.ai.embedding.BatchingStrategy; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.embedding.EmbeddingOptionsBuilder; -import org.springframework.ai.embedding.TokenCountBatchingStrategy; import org.springframework.ai.observation.conventions.VectorStoreProvider; import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric; import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder; @@ -139,6 +141,8 @@ */ public class OpenSearchVectorStore extends AbstractObservationVectorStore implements InitializingBean { + private static final Logger logger = LoggerFactory.getLogger(OpenSearchVectorStore.class); + public static final String COSINE_SIMILARITY_FUNCTION = "cosinesimil"; public static final String DEFAULT_INDEX_NAME = "spring-ai-document-index"; @@ -230,6 +234,31 @@ private BulkResponse bulkRequest(BulkRequest bulkRequest) { } } + @Override + protected void doDelete(Filter.Expression filterExpression) { + Assert.notNull(filterExpression, "Filter expression must not be null"); + + try { + String filterStr = this.filterExpressionConverter.convertExpression(filterExpression); + + // Create delete by query request + DeleteByQueryRequest request = new DeleteByQueryRequest.Builder().index(this.index) + .query(q -> q.queryString(qs -> qs.query(filterStr))) + .build(); + + DeleteByQueryResponse response = this.openSearchClient.deleteByQuery(request); + logger.debug("Deleted {} documents matching filter expression", response.deleted()); + + if (!response.failures().isEmpty()) { + throw new IllegalStateException("Failed to delete some documents: " + response.failures()); + } + } + catch (Exception e) { + logger.error("Failed to delete documents by filter: {}", e.getMessage(), e); + throw new IllegalStateException("Failed to delete documents by filter", e); + } + } + @Override public List doSimilaritySearch(SearchRequest searchRequest) { Assert.notNull(searchRequest, "The search request must not be null."); diff --git a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreIT.java b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreIT.java index fb7659bb16a..f0ad4aedc28 100644 --- a/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreIT.java +++ b/vector-stores/spring-ai-opensearch-store/src/test/java/org/springframework/ai/vectorstore/opensearch/OpenSearchVectorStoreIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,13 +19,16 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; import java.time.Duration; import java.time.ZonedDateTime; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.hc.core5.http.HttpHost; import org.awaitility.Awaitility; @@ -49,6 +52,7 @@ import org.springframework.ai.openai.api.OpenAiApi; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.ai.vectorstore.filter.Filter; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -413,6 +417,113 @@ public void searchDocumentsInTwoIndicesTest() { }); } + @Test + void deleteByFilter() { + getContextRunner().run(context -> { + OpenSearchVectorStore vectorStore = context.getBean("vectorStore", OpenSearchVectorStore.class); + + var bgDocument = new Document("1", "The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2020, "activationDate", new Date(1000))); + var nlDocument = new Document("2", "The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "NL", "activationDate", new Date(2000))); + var bgDocument2 = new Document("3", "The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2023, "activationDate", new Date(3000))); + + vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2)); + + Awaitility.await() + .until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("The World").topK(5).build()), + hasSize(3)); + + Filter.Expression filterExpression = new Filter.Expression(Filter.ExpressionType.EQ, + new Filter.Key("country"), new Filter.Value("BG")); + + vectorStore.delete(filterExpression); + + Awaitility.await() + .until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("The World").topK(5).build()), + hasSize(1)); + + List results = vectorStore + .similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build()); + + assertThat(results).hasSize(1); + assertThat(results.get(0).getMetadata()).containsEntry("country", "NL"); + }); + } + + @Test + void deleteWithStringFilterExpression() { + getContextRunner().run(context -> { + OpenSearchVectorStore vectorStore = context.getBean("vectorStore", OpenSearchVectorStore.class); + + var bgDocument = new Document("1", "The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2020, "activationDate", new Date(1000))); + var nlDocument = new Document("2", "The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "NL", "activationDate", new Date(2000))); + var bgDocument2 = new Document("3", "The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2023, "activationDate", new Date(3000))); + + vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2)); + + Awaitility.await() + .until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("The World").topK(5).build()), + hasSize(3)); + + vectorStore.delete("country == 'BG'"); + + Awaitility.await() + .until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("The World").topK(5).build()), + hasSize(1)); + + List results = vectorStore + .similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build()); + + assertThat(results).hasSize(1); + assertThat(results.get(0).getMetadata()).containsEntry("country", "NL"); + }); + } + + @Test + void deleteWithComplexFilterExpression() { + getContextRunner().run(context -> { + OpenSearchVectorStore vectorStore = context.getBean("vectorStore", OpenSearchVectorStore.class); + + var doc1 = new Document("1", "Content 1", Map.of("type", "A", "priority", 1)); + var doc2 = new Document("2", "Content 2", Map.of("type", "A", "priority", 2)); + var doc3 = new Document("3", "Content 3", Map.of("type", "B", "priority", 1)); + + vectorStore.add(List.of(doc1, doc2, doc3)); + + Awaitility.await() + .until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("Content").topK(5).build()), + hasSize(3)); + + // Complex filter expression: (type == 'A' AND priority > 1) + Filter.Expression priorityFilter = new Filter.Expression(Filter.ExpressionType.GT, + new Filter.Key("priority"), new Filter.Value(1)); + Filter.Expression typeFilter = new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("type"), + new Filter.Value("A")); + Filter.Expression complexFilter = new Filter.Expression(Filter.ExpressionType.AND, typeFilter, + priorityFilter); + + vectorStore.delete(complexFilter); + + Awaitility.await() + .until(() -> vectorStore.similaritySearch(SearchRequest.builder().query("Content").topK(5).build()), + hasSize(2)); + + var results = vectorStore + .similaritySearch(SearchRequest.builder().query("Content").topK(5).similarityThresholdAll().build()); + + assertThat(results).hasSize(2); + assertThat(results.stream().map(doc -> doc.getMetadata().get("type")).collect(Collectors.toList())) + .containsExactlyInAnyOrder("A", "B"); + assertThat(results.stream().map(doc -> doc.getMetadata().get("priority")).collect(Collectors.toList())) + .containsExactlyInAnyOrder(1, 1); + }); + } + @SpringBootConfiguration @EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class }) public static class TestApplication {