Skip to content

Commit 68a161f

Browse files
afoucretafoucret
authored
KQL: Support boolean operators in field queries (#133737) (#133836)
(cherry picked from commit 36a00d1)
1 parent 40de70e commit 68a161f

File tree

15 files changed

+703
-340
lines changed

15 files changed

+703
-340
lines changed

‎docs/changelog/133737.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 133737
2+
summary: "KQL: Support boolean operators in field queries"
3+
area: Search
4+
type: bug
5+
issues:
6+
- 132366

‎server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ private SearchCapabilities() {}
3131
private static final String TRANSFORM_RANK_RRF_TO_RETRIEVER = "transform_rank_rrf_to_retriever";
3232
/** Support kql query. */
3333
private static final String KQL_QUERY_SUPPORTED = "kql_query";
34+
private static final String KQL_QUERY_BOOLEAN_FIELD_QUERY_SUPPORTED = "kql_query_boolean_field_query";
35+
3436
/** Support propagating nested retrievers' inner_hits to top-level compound retrievers . */
3537
private static final String NESTED_RETRIEVER_INNER_HITS_SUPPORT = "nested_retriever_inner_hits_support";
3638
/** Fixed the math in {@code moving_fn}'s {@code linearWeightedAvg}. */
@@ -64,6 +66,7 @@ private SearchCapabilities() {}
6466
capabilities.add(MOVING_FN_RIGHT_MATH);
6567
capabilities.add(K_DEFAULT_TO_SIZE);
6668
capabilities.add(KQL_QUERY_SUPPORTED);
69+
capabilities.add(KQL_QUERY_BOOLEAN_FIELD_QUERY_SUPPORTED);
6770
capabilities.add(HIGHLIGHT_MAX_ANALYZED_OFFSET_DEFAULT);
6871
capabilities.add(SIGNIFICANT_TERMS_BACKGROUND_FILTER_AS_SUB);
6972
capabilities.add(SIGNIFICANT_TERMS_ON_NESTED_FIELDS);

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,12 @@ public void testKqlQueryWithinEval() {
7272
public void testInvalidKqlQueryEof() {
7373
var query = """
7474
FROM test
75-
| WHERE kql("content: ((((dog")
75+
| WHERE kql("content: (dog")
7676
""";
7777

7878
var error = expectThrows(QueryShardException.class, () -> run(query));
79-
assertThat(error.getMessage(), containsString("Failed to parse KQL query [content: ((((dog]"));
80-
assertThat(error.getRootCause().getMessage(), containsString("line 1:11: mismatched input '('"));
79+
assertThat(error.getMessage(), containsString("Failed to parse KQL query [content: (dog]"));
80+
assertThat(error.getRootCause().getMessage(), containsString("line 1:14: missing ')' at '<EOF>'"));
8181
}
8282

8383
public void testInvalidKqlQueryLexicalError() {

‎x-pack/plugin/kql/src/main/antlr/KqlBase.g4‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ topLevelQuery
2727

2828
query
2929
: <assoc=right> query operator=(AND|OR) query #booleanQuery
30-
| simpleQuery #defaultQuery
30+
| simpleQuery #defaultQuery
3131
;
3232

3333
simpleQuery
@@ -51,7 +51,7 @@ nestedQuery
5151

5252
nestedSubQuery
5353
: <assoc=right> nestedSubQuery operator=(AND|OR) nestedSubQuery #booleanNestedQuery
54-
| nestedSimpleSubQuery #defaultNestedQuery
54+
| nestedSimpleSubQuery #defaultNestedQuery
5555
;
5656

5757
nestedSimpleSubQuery
@@ -89,21 +89,27 @@ existsQuery
8989

9090
fieldQuery
9191
: fieldName COLON fieldQueryValue
92-
| fieldName COLON LEFT_PARENTHESIS fieldQueryValue RIGHT_PARENTHESIS
9392
;
9493

9594
fieldLessQuery
9695
: fieldQueryValue
97-
| LEFT_PARENTHESIS fieldQueryValue RIGHT_PARENTHESIS
9896
;
9997

10098
fieldQueryValue
101-
: (AND|OR|NOT)? (UNQUOTED_LITERAL|WILDCARD)+ (NOT|AND|OR)?
102-
| (AND|OR) (AND|OR|NOT)?
103-
| NOT (AND|OR)?
99+
: (UNQUOTED_LITERAL|WILDCARD)+
100+
| (UNQUOTED_LITERAL|WILDCARD)? (OR|AND|NOT)+
101+
| (AND|OR)+ (UNQUOTED_LITERAL|WILDCARD)?
104102
| QUOTED_STRING
103+
| operator=NOT (fieldQueryValue)?
104+
| LEFT_PARENTHESIS booleanFieldQueryValue RIGHT_PARENTHESIS
105105
;
106106

107+
booleanFieldQueryValue
108+
: booleanFieldQueryValue operator=(AND|OR) fieldQueryValue
109+
| LEFT_PARENTHESIS booleanFieldQueryValue RIGHT_PARENTHESIS
110+
| fieldQueryValue
111+
;
112+
107113
fieldName
108114
: value=UNQUOTED_LITERAL
109115
| value=QUOTED_STRING

‎x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java‎

Lines changed: 67 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.Set;
2626
import java.util.function.BiConsumer;
2727
import java.util.function.BiFunction;
28+
import java.util.function.Consumer;
2829

2930
import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
3031
import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isDateField;
@@ -207,38 +208,76 @@ public QueryBuilder visitFieldLessQuery(KqlBaseParser.FieldLessQueryContext ctx)
207208

208209
@Override
209210
public QueryBuilder visitFieldQuery(KqlBaseParser.FieldQueryContext ctx) {
211+
return parseFieldQuery(ctx.fieldName(), ctx.fieldQueryValue());
212+
}
210213

211-
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1);
212-
String queryText = extractText(ctx.fieldQueryValue());
213-
boolean hasWildcard = hasWildcard(ctx.fieldQueryValue());
214+
public QueryBuilder parseBooleanFieldQuery(
215+
KqlBaseParser.FieldNameContext fieldNameCtx,
216+
KqlBaseParser.BooleanFieldQueryValueContext booleanFieldQueryValueCtx
217+
) {
218+
if (booleanFieldQueryValueCtx.operator != null) {
219+
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
220+
221+
Token operator = booleanFieldQueryValueCtx.operator;
222+
Consumer<QueryBuilder> boolClauseConsumer = operator.getType() == KqlBaseParser.AND
223+
? boolQueryBuilder::must
224+
: boolQueryBuilder::should;
225+
boolClauseConsumer.accept(parseBooleanFieldQuery(fieldNameCtx, booleanFieldQueryValueCtx.booleanFieldQueryValue()));
226+
boolClauseConsumer.accept(parseFieldQuery(fieldNameCtx, booleanFieldQueryValueCtx.fieldQueryValue()));
227+
228+
return operator.getType() == KqlBaseParser.AND
229+
? rewriteConjunctionQuery(boolQueryBuilder)
230+
: rewriteDisjunctionQuery(boolQueryBuilder);
231+
} else if (booleanFieldQueryValueCtx.booleanFieldQueryValue() != null) {
232+
return parseBooleanFieldQuery(fieldNameCtx, booleanFieldQueryValueCtx.booleanFieldQueryValue());
233+
} else {
234+
assert booleanFieldQueryValueCtx.fieldQueryValue() != null;
235+
return parseFieldQuery(fieldNameCtx, booleanFieldQueryValueCtx.fieldQueryValue());
236+
}
237+
}
214238

215-
withFields(ctx.fieldName(), (fieldName, mappedFieldType) -> {
216-
QueryBuilder fieldQuery = null;
217-
218-
if (hasWildcard && isKeywordField(mappedFieldType)) {
219-
fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
220-
} else if (hasWildcard) {
221-
fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName);
222-
} else if (isDateField(mappedFieldType)) {
223-
RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText);
224-
if (kqlParsingContext.timeZone() != null) {
225-
rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId());
239+
public QueryBuilder parseFieldQuery(
240+
KqlBaseParser.FieldNameContext fieldNameCtx,
241+
KqlBaseParser.FieldQueryValueContext fieldQueryValueCtx
242+
) {
243+
if (fieldQueryValueCtx.operator != null) {
244+
assert fieldQueryValueCtx.fieldQueryValue() != null;
245+
return QueryBuilders.boolQuery().mustNot(parseFieldQuery(fieldNameCtx, fieldQueryValueCtx.fieldQueryValue()));
246+
} else if (fieldQueryValueCtx.booleanFieldQueryValue() != null) {
247+
return parseBooleanFieldQuery(fieldNameCtx, fieldQueryValueCtx.booleanFieldQueryValue());
248+
} else {
249+
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
250+
String queryText = extractText(fieldQueryValueCtx);
251+
boolean hasWildcard = hasWildcard(fieldQueryValueCtx);
252+
253+
withFields(fieldNameCtx, (fieldName, mappedFieldType) -> {
254+
QueryBuilder fieldQuery;
255+
256+
if (hasWildcard && isKeywordField(mappedFieldType)) {
257+
fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
258+
} else if (hasWildcard) {
259+
fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName);
260+
} else if (isDateField(mappedFieldType)) {
261+
RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText);
262+
if (kqlParsingContext.timeZone() != null) {
263+
rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId());
264+
}
265+
fieldQuery = rangeFieldQuery;
266+
} else if (isKeywordField(mappedFieldType)) {
267+
fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
268+
} else if (fieldQueryValueCtx.QUOTED_STRING() != null) {
269+
fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText);
270+
} else {
271+
fieldQuery = QueryBuilders.matchQuery(fieldName, queryText);
226272
}
227-
fieldQuery = rangeFieldQuery;
228-
} else if (isKeywordField(mappedFieldType)) {
229-
fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
230-
} else if (ctx.fieldQueryValue().QUOTED_STRING() != null) {
231-
fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText);
232-
} else {
233-
fieldQuery = QueryBuilders.matchQuery(fieldName, queryText);
234-
}
235273

236-
if (fieldQuery != null) {
237-
boolQueryBuilder.should(wrapWithNestedQuery(fieldName, fieldQuery));
238-
}
239-
});
274+
if (fieldQuery != null) {
275+
boolQueryBuilder.should(wrapWithNestedQuery(fieldName, fieldQuery));
276+
}
277+
});
240278

241-
return rewriteDisjunctionQuery(boolQueryBuilder);
279+
return rewriteDisjunctionQuery(boolQueryBuilder);
280+
}
242281
}
243282

244283
private static boolean isAndQuery(ParserRuleContext ctx) {
@@ -269,9 +308,7 @@ private void withFields(KqlBaseParser.FieldNameContext ctx, BiConsumer<String, M
269308
return;
270309
}
271310

272-
if (ctx.value.getType() == KqlBaseParser.QUOTED_STRING) {
273-
assert fieldNames.size() < 2 : "expecting only one matching field";
274-
}
311+
assert ctx.value.getType() != KqlBaseParser.QUOTED_STRING || fieldNames.size() < 2 : "expecting only one matching field";
275312

276313
fieldNames.forEach(fieldName -> {
277314
MappedFieldType fieldType = kqlParsingContext.fieldType(fieldName);

0 commit comments

Comments
 (0)