Skip to content

Commit 78c5e00

Browse files
jhuynh1sobychacko
jhuynh1
authored and
sobychacko
committed
Add filter query support for GemFireVectorStore
Signed-off-by: Jason Huynh <[email protected]> Fixed failing tests Disabling the test due to inconsistencies with testcontainers in different test environment. Signed-off-by: Nabarun Nag <[email protected]> Checkstyle fixes Fixing formatting issues Adding junit testcontainer dependency for testing Signed-off-by: Soby Chacko <[email protected]>
1 parent 9907b2c commit 78c5e00

File tree

5 files changed

+492
-9
lines changed

5 files changed

+492
-9
lines changed

‎vector-stores/spring-ai-gemfire-store/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@
9898
<artifactId>micrometer-observation-test</artifactId>
9999
<scope>test</scope>
100100
</dependency>
101+
<dependency>
102+
<groupId>org.testcontainers</groupId>
103+
<artifactId>junit-jupiter</artifactId>
104+
<scope>test</scope>
105+
</dependency>
101106
</dependencies>
102107

103108
</project>

‎vector-stores/spring-ai-gemfire-store/src/main/java/org/springframework/ai/vectorstore/GemFireAiSearchFilterExpressionConverter.java‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
* Copyright 2023-2025 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.ai.vectorstore;
18+
19+
import java.text.ParseException;
20+
import java.text.SimpleDateFormat;
21+
import java.util.Date;
22+
import java.util.List;
23+
import java.util.TimeZone;
24+
import java.util.regex.Pattern;
25+
26+
import org.springframework.ai.vectorstore.filter.Filter;
27+
import org.springframework.ai.vectorstore.filter.Filter.Expression;
28+
import org.springframework.ai.vectorstore.filter.Filter.Key;
29+
import org.springframework.ai.vectorstore.filter.converter.AbstractFilterExpressionConverter;
30+
31+
/**
32+
* GemFireAiSearchFilterExpressionConverter is a class that converts Filter.Expression
33+
* objects into GemFire VectorDB query string representation. It extends the
34+
* AbstractFilter ExpressionConverter class.
35+
*
36+
* @author Jason Huynh
37+
*/
38+
public class GemFireAiSearchFilterExpressionConverter extends AbstractFilterExpressionConverter {
39+
40+
private static final Pattern DATE_FORMAT_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z");
41+
42+
private final SimpleDateFormat dateFormat;
43+
44+
public GemFireAiSearchFilterExpressionConverter() {
45+
this.dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
46+
this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
47+
}
48+
49+
@Override
50+
protected void doExpression(Expression expression, StringBuilder context) {
51+
if (expression.type() == Filter.ExpressionType.IN || expression.type() == Filter.ExpressionType.NIN) {
52+
context.append(getOperationSymbol(expression));
53+
this.convertOperand(expression.left(), context);
54+
context.append("(");
55+
this.convertOperand(expression.right(), context);
56+
context.append(")");
57+
}
58+
else if (expression.type() == Filter.ExpressionType.GT || expression.type() == Filter.ExpressionType.GTE) {
59+
this.convertOperand(expression.left(), context);
60+
context.append(getOperationSymbol(expression));
61+
this.convertOperand(expression.right(), context);
62+
context.append(" TO *]");
63+
}
64+
else if (expression.type() == Filter.ExpressionType.LT || expression.type() == Filter.ExpressionType.LTE) {
65+
this.convertOperand(expression.left(), context);
66+
context.append("[* TO ");
67+
this.convertOperand(expression.right(), context);
68+
context.append(getOperationSymbol(expression));
69+
}
70+
else {
71+
this.convertOperand(expression.left(), context);
72+
context.append(getOperationSymbol(expression));
73+
this.convertOperand(expression.right(), context);
74+
}
75+
}
76+
77+
@Override
78+
protected void doStartValueRange(Filter.Value listValue, StringBuilder context) {
79+
}
80+
81+
@Override
82+
protected void doEndValueRange(Filter.Value listValue, StringBuilder context) {
83+
}
84+
85+
@Override
86+
protected void doAddValueRangeSpitter(Filter.Value listValue, StringBuilder context) {
87+
context.append(" OR ");
88+
}
89+
90+
private String getOperationSymbol(Expression exp) {
91+
return switch (exp.type()) {
92+
case AND -> " AND ";
93+
case OR -> " OR ";
94+
case EQ, IN -> "";
95+
case NE -> " NOT ";
96+
case LT -> "}";
97+
case LTE -> "]";
98+
case GT -> "{";
99+
case GTE -> "[";
100+
case NIN -> "NOT ";
101+
default -> throw new RuntimeException("Not supported expression type: " + exp.type());
102+
};
103+
}
104+
105+
@Override
106+
public void doKey(Key key, StringBuilder context) {
107+
var identifier = hasOuterQuotes(key.key()) ? removeOuterQuotes(key.key()) : key.key();
108+
context.append(identifier.trim()).append(":");
109+
}
110+
111+
@Override
112+
protected void doValue(Filter.Value filterValue, StringBuilder context) {
113+
if (filterValue.value() instanceof List list) {
114+
int c = 0;
115+
for (Object v : list) {
116+
context.append(v);
117+
if (c++ < list.size() - 1) {
118+
this.doAddValueRangeSpitter(filterValue, context);
119+
}
120+
}
121+
}
122+
else {
123+
this.doSingleValue(filterValue.value(), context);
124+
}
125+
}
126+
127+
@Override
128+
protected void doSingleValue(Object value, StringBuilder context) {
129+
if (value instanceof Date date) {
130+
context.append(this.dateFormat.format(date));
131+
}
132+
else if (value instanceof String text) {
133+
if (DATE_FORMAT_PATTERN.matcher(text).matches()) {
134+
try {
135+
Date date = this.dateFormat.parse(text);
136+
context.append(this.dateFormat.format(date));
137+
}
138+
catch (ParseException e) {
139+
throw new IllegalArgumentException("Invalid date type:" + text, e);
140+
}
141+
}
142+
else {
143+
context.append(text);
144+
}
145+
}
146+
else {
147+
context.append(value);
148+
}
149+
}
150+
151+
@Override
152+
public void doStartGroup(Filter.Group group, StringBuilder context) {
153+
context.append("(");
154+
}
155+
156+
@Override
157+
public void doEndGroup(Filter.Group group, StringBuilder context) {
158+
context.append(")");
159+
}
160+
161+
}

‎vector-stores/spring-ai-gemfire-store/src/main/java/org/springframework/ai/vectorstore/gemfire/GemFireVectorStore.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@
3636
import org.springframework.ai.observation.conventions.VectorStoreProvider;
3737
import org.springframework.ai.util.JacksonUtils;
3838
import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder;
39+
import org.springframework.ai.vectorstore.GemFireAiSearchFilterExpressionConverter;
3940
import org.springframework.ai.vectorstore.SearchRequest;
41+
import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
4042
import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
4143
import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
4244
import org.springframework.beans.factory.InitializingBean;
@@ -114,6 +116,8 @@ public class GemFireVectorStore extends AbstractObservationVectorStore implement
114116

115117
private final String[] fields;
116118

119+
private final FilterExpressionConverter filterExpressionConverter;
120+
117121
/**
118122
* Protected constructor that accepts a builder instance. This is the preferred way to
119123
* create new GemFireVectorStore instances.
@@ -134,6 +138,7 @@ protected GemFireVectorStore(Builder builder) {
134138
.build(builder.sslEnabled ? "s" : "", builder.host, builder.port)
135139
.toString();
136140
this.client = WebClient.create(base);
141+
this.filterExpressionConverter = new GemFireAiSearchFilterExpressionConverter();
137142
this.objectMapper = JsonMapper.builder().addModules(JacksonUtils.instantiateAvailableModules()).build();
138143
}
139144

@@ -244,15 +249,16 @@ public void doDelete(List<String> idList) {
244249

245250
@Override
246251
public List<Document> doSimilaritySearch(SearchRequest request) {
252+
String filterQuery = null;
247253
if (request.hasFilterExpression()) {
248-
throw new UnsupportedOperationException("GemFire currently does not support metadata filter expressions.");
254+
filterQuery = this.filterExpressionConverter.convertExpression(request.getFilterExpression());
249255
}
250256
float[] floatVector = this.embeddingModel.embed(request.getQuery());
251257
return this.client.post()
252258
.uri("/" + this.indexName + QUERY)
253259
.contentType(MediaType.APPLICATION_JSON)
254260
.bodyValue(new QueryRequest(floatVector, request.getTopK(), request.getTopK(), // TopKPerBucket
255-
true))
261+
true, filterQuery))
256262
.retrieve()
257263
.bodyToFlux(QueryResponse.class)
258264
.filter(r -> r.score >= request.getSimilarityThreshold())
@@ -473,11 +479,20 @@ private static final class QueryRequest {
473479
@JsonProperty("include-metadata")
474480
private final boolean includeMetadata;
475481

482+
@JsonProperty("filter-query")
483+
@JsonInclude(JsonInclude.Include.NON_NULL)
484+
private final String filterQuery;
485+
476486
QueryRequest(float[] vector, int k, int kPerBucket, boolean includeMetadata) {
487+
this(vector, k, kPerBucket, includeMetadata, null);
488+
}
489+
490+
QueryRequest(float[] vector, int k, int kPerBucket, boolean includeMetadata, String filterQuery) {
477491
this.vector = vector;
478492
this.k = k;
479493
this.kPerBucket = kPerBucket;
480494
this.includeMetadata = includeMetadata;
495+
this.filterQuery = filterQuery;
481496
}
482497

483498
public float[] getVector() {
@@ -496,6 +511,10 @@ public boolean isIncludeMetadata() {
496511
return this.includeMetadata;
497512
}
498513

514+
public String getFilterQuery() {
515+
return this.filterQuery;
516+
}
517+
499518
}
500519

501520
private static final class QueryResponse {

‎vector-stores/spring-ai-gemfire-store/src/test/java/org/springframework/ai/vectorstore/GemFireAiSearchFilterExpressionConverterTest.java‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/*
2+
* Copyright 2023-2025 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.ai.vectorstore;
18+
19+
import java.util.Date;
20+
import java.util.List;
21+
22+
import org.junit.jupiter.api.Test;
23+
24+
import org.springframework.ai.vectorstore.filter.Filter;
25+
import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
26+
27+
import static org.assertj.core.api.Assertions.assertThat;
28+
import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.AND;
29+
import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.EQ;
30+
import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.GT;
31+
import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.GTE;
32+
import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.IN;
33+
import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.LTE;
34+
import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NE;
35+
import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.NIN;
36+
import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.OR;
37+
38+
/**
39+
* @author Jason Huynh
40+
*/
41+
class GemFireAiSearchFilterExpressionConverterTest {
42+
43+
final FilterExpressionConverter converter = new GemFireAiSearchFilterExpressionConverter();
44+
45+
@Test
46+
public void testDate() {
47+
String vectorExpr = this.converter.convertExpression(new Filter.Expression(EQ, new Filter.Key("activationDate"),
48+
new Filter.Value(new Date(1704637752148L))));
49+
assertThat(vectorExpr).isEqualTo("activationDate:2024-01-07T14:29:12Z");
50+
51+
vectorExpr = this.converter.convertExpression(
52+
new Filter.Expression(EQ, new Filter.Key("activationDate"), new Filter.Value("1970-01-01T00:00:02Z")));
53+
assertThat(vectorExpr).isEqualTo("activationDate:1970-01-01T00:00:02Z");
54+
}
55+
56+
@Test
57+
public void testEQ() {
58+
String vectorExpr = this.converter
59+
.convertExpression(new Filter.Expression(EQ, new Filter.Key("country"), new Filter.Value("BG")));
60+
assertThat(vectorExpr).isEqualTo("country:BG");
61+
}
62+
63+
@Test
64+
public void testEqAndGte() {
65+
String vectorExpr = this.converter.convertExpression(new Filter.Expression(AND,
66+
new Filter.Expression(EQ, new Filter.Key("genre"), new Filter.Value("drama")),
67+
new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020))));
68+
assertThat(vectorExpr).isEqualTo("genre:drama AND year:[2020 TO *]");
69+
}
70+
71+
@Test
72+
public void testEqAndGe() {
73+
String vectorExpr = this.converter.convertExpression(new Filter.Expression(AND,
74+
new Filter.Expression(EQ, new Filter.Key("genre"), new Filter.Value("drama")),
75+
new Filter.Expression(GT, new Filter.Key("year"), new Filter.Value(2020))));
76+
assertThat(vectorExpr).isEqualTo("genre:drama AND year:{2020 TO *]");
77+
}
78+
79+
@Test
80+
public void testIn() {
81+
String vectorExpr = this.converter.convertExpression(new Filter.Expression(IN, new Filter.Key("genre"),
82+
new Filter.Value(List.of("comedy", "documentary", "drama"))));
83+
assertThat(vectorExpr).isEqualTo("genre:(comedy OR documentary OR drama)");
84+
}
85+
86+
@Test
87+
public void testNe() {
88+
String vectorExpr = this.converter.convertExpression(
89+
new Filter.Expression(OR, new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020)),
90+
new Filter.Expression(AND,
91+
new Filter.Expression(EQ, new Filter.Key("country"), new Filter.Value("BG")),
92+
new Filter.Expression(NE, new Filter.Key("city"), new Filter.Value("Sofia")))));
93+
assertThat(vectorExpr).isEqualTo("year:[2020 TO *] OR country:BG AND city: NOT Sofia");
94+
}
95+
96+
@Test
97+
public void testGroup() {
98+
String vectorExpr = this.converter.convertExpression(new Filter.Expression(AND,
99+
new Filter.Group(new Filter.Expression(OR,
100+
new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020)),
101+
new Filter.Expression(EQ, new Filter.Key("country"), new Filter.Value("BG")))),
102+
new Filter.Expression(NIN, new Filter.Key("city"), new Filter.Value(List.of("Sofia", "Plovdiv")))));
103+
assertThat(vectorExpr).isEqualTo("(year:[2020 TO *] OR country:BG) AND NOT city:(Sofia OR Plovdiv)");
104+
}
105+
106+
@Test
107+
public void testBoolean() {
108+
String vectorExpr = this.converter.convertExpression(new Filter.Expression(AND,
109+
new Filter.Expression(AND, new Filter.Expression(EQ, new Filter.Key("isOpen"), new Filter.Value(true)),
110+
new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020))),
111+
new Filter.Expression(IN, new Filter.Key("country"), new Filter.Value(List.of("BG", "NL", "US")))));
112+
113+
assertThat(vectorExpr).isEqualTo("isOpen:true AND year:[2020 TO *] AND country:(BG OR NL OR US)");
114+
}
115+
116+
@Test
117+
public void testDecimal() {
118+
String vectorExpr = this.converter.convertExpression(new Filter.Expression(AND,
119+
new Filter.Expression(GTE, new Filter.Key("temperature"), new Filter.Value(-15.6)),
120+
new Filter.Expression(LTE, new Filter.Key("temperature"), new Filter.Value(20.13))));
121+
122+
assertThat(vectorExpr).isEqualTo("temperature:[-15.6 TO *] AND temperature:[* TO 20.13]");
123+
}
124+
125+
@Test
126+
public void testComplexIdentifiers() {
127+
String vectorExpr = this.converter
128+
.convertExpression(new Filter.Expression(EQ, new Filter.Key("\"country 1 2 3\""), new Filter.Value("BG")));
129+
assertThat(vectorExpr).isEqualTo("country 1 2 3:BG");
130+
131+
vectorExpr = this.converter
132+
.convertExpression(new Filter.Expression(EQ, new Filter.Key("'country 1 2 3'"), new Filter.Value("BG")));
133+
assertThat(vectorExpr).isEqualTo("country 1 2 3:BG");
134+
}
135+
136+
}

0 commit comments

Comments
 (0)