Skip to content

[9.0] KQL: Support boolean operators in field queries (#133737) #133836

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 29, 2025
Merged

[9.0] KQL: Support boolean operators in field queries (#133737) #133836

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/changelog/133737.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 133737
summary: "KQL: Support boolean operators in field queries"
area: Search
type: bug
issues:
- 132366
3 changes: 3 additions & 0 deletions server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ private SearchCapabilities() {}
private static final String TRANSFORM_RANK_RRF_TO_RETRIEVER = "transform_rank_rrf_to_retriever";
/** Support kql query. */
private static final String KQL_QUERY_SUPPORTED = "kql_query";
private static final String KQL_QUERY_BOOLEAN_FIELD_QUERY_SUPPORTED = "kql_query_boolean_field_query";

/** Support propagating nested retrievers' inner_hits to top-level compound retrievers . */
private static final String NESTED_RETRIEVER_INNER_HITS_SUPPORT = "nested_retriever_inner_hits_support";
/** Fixed the math in {@code moving_fn}'s {@code linearWeightedAvg}. */
Expand Down Expand Up @@ -64,6 +66,7 @@ private SearchCapabilities() {}
capabilities.add(MOVING_FN_RIGHT_MATH);
capabilities.add(K_DEFAULT_TO_SIZE);
capabilities.add(KQL_QUERY_SUPPORTED);
capabilities.add(KQL_QUERY_BOOLEAN_FIELD_QUERY_SUPPORTED);
capabilities.add(HIGHLIGHT_MAX_ANALYZED_OFFSET_DEFAULT);
capabilities.add(SIGNIFICANT_TERMS_BACKGROUND_FILTER_AS_SUB);
capabilities.add(SIGNIFICANT_TERMS_ON_NESTED_FIELDS);
Expand Down
6 changes: 3 additions & 3 deletions .../esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,12 @@ public void testKqlQueryWithinEval() {
public void testInvalidKqlQueryEof() {
var query = """
FROM test
| WHERE kql("content: ((((dog")
| WHERE kql("content: (dog")
""";

var error = expectThrows(QueryShardException.class, () -> run(query));
assertThat(error.getMessage(), containsString("Failed to parse KQL query [content: ((((dog]"));
assertThat(error.getRootCause().getMessage(), containsString("line 1:11: mismatched input '('"));
assertThat(error.getMessage(), containsString("Failed to parse KQL query [content: (dog]"));
assertThat(error.getRootCause().getMessage(), containsString("line 1:14: missing ')' at '<EOF>'"));
}

public void testInvalidKqlQueryLexicalError() {
Expand Down
20 changes: 13 additions & 7 deletions x-pack/plugin/kql/src/main/antlr/KqlBase.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ topLevelQuery

query
: <assoc=right> query operator=(AND|OR) query #booleanQuery
| simpleQuery #defaultQuery
| simpleQuery #defaultQuery
;

simpleQuery
Expand All @@ -51,7 +51,7 @@ nestedQuery

nestedSubQuery
: <assoc=right> nestedSubQuery operator=(AND|OR) nestedSubQuery #booleanNestedQuery
| nestedSimpleSubQuery #defaultNestedQuery
| nestedSimpleSubQuery #defaultNestedQuery
;

nestedSimpleSubQuery
Expand Down Expand Up @@ -89,21 +89,27 @@ existsQuery

fieldQuery
: fieldName COLON fieldQueryValue
| fieldName COLON LEFT_PARENTHESIS fieldQueryValue RIGHT_PARENTHESIS
;

fieldLessQuery
: fieldQueryValue
| LEFT_PARENTHESIS fieldQueryValue RIGHT_PARENTHESIS
;

fieldQueryValue
: (AND|OR|NOT)? (UNQUOTED_LITERAL|WILDCARD)+ (NOT|AND|OR)?
| (AND|OR) (AND|OR|NOT)?
| NOT (AND|OR)?
: (UNQUOTED_LITERAL|WILDCARD)+
| (UNQUOTED_LITERAL|WILDCARD)? (OR|AND|NOT)+
| (AND|OR)+ (UNQUOTED_LITERAL|WILDCARD)?
| QUOTED_STRING
| operator=NOT (fieldQueryValue)?
| LEFT_PARENTHESIS booleanFieldQueryValue RIGHT_PARENTHESIS
;

booleanFieldQueryValue
: booleanFieldQueryValue operator=(AND|OR) fieldQueryValue
| LEFT_PARENTHESIS booleanFieldQueryValue RIGHT_PARENTHESIS
| fieldQueryValue
;

fieldName
: value=UNQUOTED_LITERAL
| value=QUOTED_STRING
Expand Down
97 changes: 67 additions & 30 deletions x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Consumer;

import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isDateField;
Expand Down Expand Up @@ -207,38 +208,76 @@ public QueryBuilder visitFieldLessQuery(KqlBaseParser.FieldLessQueryContext ctx)

@Override
public QueryBuilder visitFieldQuery(KqlBaseParser.FieldQueryContext ctx) {
return parseFieldQuery(ctx.fieldName(), ctx.fieldQueryValue());
}

BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1);
String queryText = extractText(ctx.fieldQueryValue());
boolean hasWildcard = hasWildcard(ctx.fieldQueryValue());
public QueryBuilder parseBooleanFieldQuery(
KqlBaseParser.FieldNameContext fieldNameCtx,
KqlBaseParser.BooleanFieldQueryValueContext booleanFieldQueryValueCtx
) {
if (booleanFieldQueryValueCtx.operator != null) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();

Token operator = booleanFieldQueryValueCtx.operator;
Consumer<QueryBuilder> boolClauseConsumer = operator.getType() == KqlBaseParser.AND
? boolQueryBuilder::must
: boolQueryBuilder::should;
boolClauseConsumer.accept(parseBooleanFieldQuery(fieldNameCtx, booleanFieldQueryValueCtx.booleanFieldQueryValue()));
boolClauseConsumer.accept(parseFieldQuery(fieldNameCtx, booleanFieldQueryValueCtx.fieldQueryValue()));

return operator.getType() == KqlBaseParser.AND
? rewriteConjunctionQuery(boolQueryBuilder)
: rewriteDisjunctionQuery(boolQueryBuilder);
} else if (booleanFieldQueryValueCtx.booleanFieldQueryValue() != null) {
return parseBooleanFieldQuery(fieldNameCtx, booleanFieldQueryValueCtx.booleanFieldQueryValue());
} else {
assert booleanFieldQueryValueCtx.fieldQueryValue() != null;
return parseFieldQuery(fieldNameCtx, booleanFieldQueryValueCtx.fieldQueryValue());
}
}

withFields(ctx.fieldName(), (fieldName, mappedFieldType) -> {
QueryBuilder fieldQuery = null;

if (hasWildcard && isKeywordField(mappedFieldType)) {
fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
} else if (hasWildcard) {
fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName);
} else if (isDateField(mappedFieldType)) {
RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText);
if (kqlParsingContext.timeZone() != null) {
rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId());
public QueryBuilder parseFieldQuery(
KqlBaseParser.FieldNameContext fieldNameCtx,
KqlBaseParser.FieldQueryValueContext fieldQueryValueCtx
) {
if (fieldQueryValueCtx.operator != null) {
assert fieldQueryValueCtx.fieldQueryValue() != null;
return QueryBuilders.boolQuery().mustNot(parseFieldQuery(fieldNameCtx, fieldQueryValueCtx.fieldQueryValue()));
} else if (fieldQueryValueCtx.booleanFieldQueryValue() != null) {
return parseBooleanFieldQuery(fieldNameCtx, fieldQueryValueCtx.booleanFieldQueryValue());
} else {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
String queryText = extractText(fieldQueryValueCtx);
boolean hasWildcard = hasWildcard(fieldQueryValueCtx);

withFields(fieldNameCtx, (fieldName, mappedFieldType) -> {
QueryBuilder fieldQuery;

if (hasWildcard && isKeywordField(mappedFieldType)) {
fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
} else if (hasWildcard) {
fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName);
} else if (isDateField(mappedFieldType)) {
RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText);
if (kqlParsingContext.timeZone() != null) {
rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId());
}
fieldQuery = rangeFieldQuery;
} else if (isKeywordField(mappedFieldType)) {
fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
} else if (fieldQueryValueCtx.QUOTED_STRING() != null) {
fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText);
} else {
fieldQuery = QueryBuilders.matchQuery(fieldName, queryText);
}
fieldQuery = rangeFieldQuery;
} else if (isKeywordField(mappedFieldType)) {
fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
} else if (ctx.fieldQueryValue().QUOTED_STRING() != null) {
fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText);
} else {
fieldQuery = QueryBuilders.matchQuery(fieldName, queryText);
}

if (fieldQuery != null) {
boolQueryBuilder.should(wrapWithNestedQuery(fieldName, fieldQuery));
}
});
if (fieldQuery != null) {
boolQueryBuilder.should(wrapWithNestedQuery(fieldName, fieldQuery));
}
});

return rewriteDisjunctionQuery(boolQueryBuilder);
return rewriteDisjunctionQuery(boolQueryBuilder);
}
}

private static boolean isAndQuery(ParserRuleContext ctx) {
Expand Down Expand Up @@ -269,9 +308,7 @@ private void withFields(KqlBaseParser.FieldNameContext ctx, BiConsumer<String, M
return;
}

if (ctx.value.getType() == KqlBaseParser.QUOTED_STRING) {
assert fieldNames.size() < 2 : "expecting only one matching field";
}
assert ctx.value.getType() != KqlBaseParser.QUOTED_STRING || fieldNames.size() < 2 : "expecting only one matching field";

fieldNames.forEach(fieldName -> {
MappedFieldType fieldType = kqlParsingContext.fieldType(fieldName);
Expand Down
Loading