Skip to content

Commit 347b7fe

Browse files
afoucretafoucret
authored
[8.x] Add kql query to the DSL (#116262) (#116482)
* Add kql query to the DSL (#116262) (cherry picked from commit e2c29f5) # Conflicts: # server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java * Fix typo introduced during merge.
1 parent b7e96f1 commit 347b7fe

File tree

20 files changed

+1846
-229
lines changed

20 files changed

+1846
-229
lines changed

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ static TransportVersion def(int id) {
188188
public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_00_0);
189189
public static final TransportVersion LOGSDB_TELEMETRY = def(8_784_00_0);
190190
public static final TransportVersion LOGSDB_TELEMETRY_STATS = def(8_785_00_0);
191+
public static final TransportVersion KQL_QUERY_ADDED = def(8_786_00_0);
191192

192193
/*
193194
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@
99

1010
package org.elasticsearch.rest.action.search;
1111

12+
import org.elasticsearch.Build;
13+
import org.elasticsearch.common.util.set.Sets;
14+
15+
import java.util.Collections;
1216
import java.util.Set;
1317

1418
/**
@@ -24,10 +28,26 @@ private SearchCapabilities() {}
2428
private static final String BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY = "bit_dense_vector_synthetic_source";
2529
/** Support Byte and Float with Bit dot product. */
2630
private static final String BYTE_FLOAT_BIT_DOT_PRODUCT_CAPABILITY = "byte_float_bit_dot_product";
31+
/** Support kql query. */
32+
private static final String KQL_QUERY_SUPPORTED = "kql_query";
33+
34+
public static final Set<String> CAPABILITIES = capabilities();
35+
36+
private static Set<String> capabilities() {
37+
Set<String> capabilities = Set.of(
38+
RANGE_REGEX_INTERVAL_QUERY_CAPABILITY,
39+
BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY,
40+
BYTE_FLOAT_BIT_DOT_PRODUCT_CAPABILITY
41+
);
42+
43+
if (Build.current().isSnapshot()) {
44+
return Collections.unmodifiableSet(Sets.union(capabilities, snapshotBuildCapabilities()));
45+
}
46+
47+
return capabilities;
48+
}
2749

28-
public static final Set<String> CAPABILITIES = Set.of(
29-
RANGE_REGEX_INTERVAL_QUERY_CAPABILITY,
30-
BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY,
31-
BYTE_FLOAT_BIT_DOT_PRODUCT_CAPABILITY
32-
);
50+
private static Set<String> snapshotBuildCapabilities() {
51+
return Set.of(KQL_QUERY_SUPPORTED);
52+
}
3353
}

‎x-pack/plugin/kql/build.gradle‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import org.elasticsearch.gradle.internal.info.BuildParams
2+
23
import static org.elasticsearch.gradle.util.PlatformUtils.normalize
34

45
apply plugin: 'elasticsearch.internal-es-plugin'
56
apply plugin: 'elasticsearch.internal-cluster-test'
7+
apply plugin: 'elasticsearch.internal-yaml-rest-test'
68
apply plugin: 'elasticsearch.publish'
79

810
esplugin {
@@ -17,19 +19,21 @@ base {
1719

1820
dependencies {
1921
compileOnly project(path: xpackModule('core'))
20-
api "org.antlr:antlr4-runtime:${versions.antlr4}"
22+
implementation "org.antlr:antlr4-runtime:${versions.antlr4}"
2123

2224
testImplementation "org.antlr:antlr4-runtime:${versions.antlr4}"
2325
testImplementation project(':test:framework')
2426
testImplementation(testArtifact(project(xpackModule('core'))))
2527
}
2628

27-
/****************************************************************
28-
* Enable QA/rest integration tests for snapshot builds only *
29-
* TODO: Enable for all builds upon this feature release *
30-
****************************************************************/
31-
if (BuildParams.isSnapshotBuild()) {
32-
addQaCheckDependencies(project)
29+
tasks.named('yamlRestTest') {
30+
usesDefaultDistribution()
31+
}.configure {
32+
/****************************************************************
33+
* Enable QA/rest integration tests for snapshot builds only *
34+
* TODO: Enable for all builds upon this feature release *
35+
****************************************************************/
36+
enabled = BuildParams.isSnapshotBuild()
3337
}
3438

3539
/**********************************

‎x-pack/plugin/kql/src/main/antlr/KqlBase.g4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ fieldQueryValue
8888
;
8989

9090
fieldName
91-
: value=UNQUOTED_LITERAL+
91+
: value=UNQUOTED_LITERAL
9292
| value=QUOTED_STRING
9393
| value=WILDCARD
9494
;

‎x-pack/plugin/kql/src/main/java/module-info.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@
1616

1717
exports org.elasticsearch.xpack.kql;
1818
exports org.elasticsearch.xpack.kql.parser;
19+
exports org.elasticsearch.xpack.kql.query;
1920
}

‎x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,21 @@
77

88
package org.elasticsearch.xpack.kql;
99

10+
import org.elasticsearch.Build;
1011
import org.elasticsearch.plugins.ExtensiblePlugin;
1112
import org.elasticsearch.plugins.Plugin;
1213
import org.elasticsearch.plugins.SearchPlugin;
14+
import org.elasticsearch.xpack.kql.query.KqlQueryBuilder;
15+
16+
import java.util.List;
1317

1418
public class KqlPlugin extends Plugin implements SearchPlugin, ExtensiblePlugin {
19+
@Override
20+
public List<QuerySpec<?>> getQueries() {
21+
if (Build.current().isSnapshot()) {
22+
return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent));
23+
}
1524

25+
return List.of();
26+
}
1627
}

‎x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java‎

Lines changed: 69 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -9,36 +9,42 @@
99

1010
import org.antlr.v4.runtime.ParserRuleContext;
1111
import org.antlr.v4.runtime.Token;
12+
import org.elasticsearch.common.regex.Regex;
1213
import org.elasticsearch.index.mapper.MappedFieldType;
1314
import org.elasticsearch.index.query.BoolQueryBuilder;
1415
import org.elasticsearch.index.query.MatchAllQueryBuilder;
1516
import org.elasticsearch.index.query.MatchNoneQueryBuilder;
1617
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
1718
import org.elasticsearch.index.query.QueryBuilder;
1819
import org.elasticsearch.index.query.QueryBuilders;
20+
import org.elasticsearch.index.query.QueryStringQueryBuilder;
1921
import org.elasticsearch.index.query.RangeQueryBuilder;
2022

23+
import java.util.Set;
2124
import java.util.function.BiConsumer;
2225
import java.util.function.BiFunction;
2326

2427
import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
25-
import static org.elasticsearch.xpack.kql.parser.KqlParserExecutionContext.isDateField;
26-
import static org.elasticsearch.xpack.kql.parser.KqlParserExecutionContext.isKeywordField;
27-
import static org.elasticsearch.xpack.kql.parser.KqlParserExecutionContext.isRuntimeField;
28+
import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isDateField;
29+
import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isKeywordField;
30+
import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isRuntimeField;
31+
import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isSearchableField;
2832
import static org.elasticsearch.xpack.kql.parser.ParserUtils.escapeLuceneQueryString;
33+
import static org.elasticsearch.xpack.kql.parser.ParserUtils.extractText;
2934
import static org.elasticsearch.xpack.kql.parser.ParserUtils.hasWildcard;
35+
import static org.elasticsearch.xpack.kql.parser.ParserUtils.typedParsing;
3036

3137
class KqlAstBuilder extends KqlBaseBaseVisitor<QueryBuilder> {
32-
private final KqlParserExecutionContext kqlParserExecutionContext;
38+
private final KqlParsingContext kqlParsingContext;
3339

34-
KqlAstBuilder(KqlParserExecutionContext kqlParserExecutionContext) {
35-
this.kqlParserExecutionContext = kqlParserExecutionContext;
40+
KqlAstBuilder(KqlParsingContext kqlParsingContext) {
41+
this.kqlParsingContext = kqlParsingContext;
3642
}
3743

3844
public QueryBuilder toQueryBuilder(ParserRuleContext ctx) {
3945
if (ctx instanceof KqlBaseParser.TopLevelQueryContext topLeveQueryContext) {
4046
if (topLeveQueryContext.query() != null) {
41-
return ParserUtils.typedParsing(this, topLeveQueryContext.query(), QueryBuilder.class);
47+
return typedParsing(this, topLeveQueryContext.query(), QueryBuilder.class);
4248
}
4349

4450
return new MatchAllQueryBuilder();
@@ -59,9 +65,9 @@ public QueryBuilder visitAndBooleanQuery(KqlBaseParser.BooleanQueryContext ctx)
5965
// TODO: KQLContext has an option to wrap the clauses into a filter instead of a must clause. Do we need it?
6066
for (ParserRuleContext subQueryCtx : ctx.query()) {
6167
if (subQueryCtx instanceof KqlBaseParser.BooleanQueryContext booleanSubQueryCtx && isAndQuery(booleanSubQueryCtx)) {
62-
ParserUtils.typedParsing(this, subQueryCtx, BoolQueryBuilder.class).must().forEach(builder::must);
68+
typedParsing(this, subQueryCtx, BoolQueryBuilder.class).must().forEach(builder::must);
6369
} else {
64-
builder.must(ParserUtils.typedParsing(this, subQueryCtx, QueryBuilder.class));
70+
builder.must(typedParsing(this, subQueryCtx, QueryBuilder.class));
6571
}
6672
}
6773

@@ -73,9 +79,9 @@ public QueryBuilder visitOrBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) {
7379

7480
for (ParserRuleContext subQueryCtx : ctx.query()) {
7581
if (subQueryCtx instanceof KqlBaseParser.BooleanQueryContext booleanSubQueryCtx && isOrQuery(booleanSubQueryCtx)) {
76-
ParserUtils.typedParsing(this, subQueryCtx, BoolQueryBuilder.class).should().forEach(builder::should);
82+
typedParsing(this, subQueryCtx, BoolQueryBuilder.class).should().forEach(builder::should);
7783
} else {
78-
builder.should(ParserUtils.typedParsing(this, subQueryCtx, QueryBuilder.class));
84+
builder.should(typedParsing(this, subQueryCtx, QueryBuilder.class));
7985
}
8086
}
8187

@@ -84,12 +90,12 @@ public QueryBuilder visitOrBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) {
8490

8591
@Override
8692
public QueryBuilder visitNotQuery(KqlBaseParser.NotQueryContext ctx) {
87-
return QueryBuilders.boolQuery().mustNot(ParserUtils.typedParsing(this, ctx.simpleQuery(), QueryBuilder.class));
93+
return QueryBuilders.boolQuery().mustNot(typedParsing(this, ctx.simpleQuery(), QueryBuilder.class));
8894
}
8995

9096
@Override
9197
public QueryBuilder visitParenthesizedQuery(KqlBaseParser.ParenthesizedQueryContext ctx) {
92-
return ParserUtils.typedParsing(this, ctx.query(), QueryBuilder.class);
98+
return typedParsing(this, ctx.query(), QueryBuilder.class);
9399
}
94100

95101
@Override
@@ -121,12 +127,16 @@ public QueryBuilder visitExistsQuery(KqlBaseParser.ExistsQueryContext ctx) {
121127
public QueryBuilder visitRangeQuery(KqlBaseParser.RangeQueryContext ctx) {
122128
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1);
123129

124-
String queryText = ParserUtils.extractText(ctx.rangeQueryValue());
130+
String queryText = extractText(ctx.rangeQueryValue());
125131
BiFunction<RangeQueryBuilder, String, RangeQueryBuilder> rangeOperation = rangeOperation(ctx.operator);
126132

127133
withFields(ctx.fieldName(), (fieldName, mappedFieldType) -> {
128134
RangeQueryBuilder rangeQuery = rangeOperation.apply(QueryBuilders.rangeQuery(fieldName), queryText);
129-
// TODO: add timezone for date fields
135+
136+
if (kqlParsingContext.timeZone() != null) {
137+
rangeQuery.timeZone(kqlParsingContext.timeZone().getId());
138+
}
139+
130140
boolQueryBuilder.should(rangeQuery);
131141
});
132142

@@ -135,42 +145,54 @@ public QueryBuilder visitRangeQuery(KqlBaseParser.RangeQueryContext ctx) {
135145

136146
@Override
137147
public QueryBuilder visitFieldLessQuery(KqlBaseParser.FieldLessQueryContext ctx) {
138-
String queryText = ParserUtils.extractText(ctx.fieldQueryValue());
148+
String queryText = extractText(ctx.fieldQueryValue());
139149

140150
if (hasWildcard(ctx.fieldQueryValue())) {
141-
// TODO: set default fields.
142-
return QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true));
151+
QueryStringQueryBuilder queryString = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true));
152+
if (kqlParsingContext.defaultField() != null) {
153+
queryString.defaultField(kqlParsingContext.defaultField());
154+
}
155+
return queryString;
143156
}
144157

145158
boolean isPhraseMatch = ctx.fieldQueryValue().QUOTED_STRING() != null;
146159

147-
return QueryBuilders.multiMatchQuery(queryText)
148-
// TODO: add default fields?
160+
MultiMatchQueryBuilder multiMatchQuery = QueryBuilders.multiMatchQuery(queryText)
149161
.type(isPhraseMatch ? MultiMatchQueryBuilder.Type.PHRASE : MultiMatchQueryBuilder.Type.BEST_FIELDS)
150162
.lenient(true);
163+
164+
if (kqlParsingContext.defaultField() != null) {
165+
kqlParsingContext.resolveDefaultFieldNames()
166+
.stream()
167+
.filter(kqlParsingContext::isSearchableField)
168+
.forEach(multiMatchQuery::field);
169+
}
170+
171+
return multiMatchQuery;
151172
}
152173

153174
@Override
154175
public QueryBuilder visitFieldQuery(KqlBaseParser.FieldQueryContext ctx) {
155176

156177
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1);
157-
String queryText = ParserUtils.extractText(ctx.fieldQueryValue());
178+
String queryText = extractText(ctx.fieldQueryValue());
158179
boolean hasWildcard = hasWildcard(ctx.fieldQueryValue());
159180

160181
withFields(ctx.fieldName(), (fieldName, mappedFieldType) -> {
161182
QueryBuilder fieldQuery = null;
162183

163184
if (hasWildcard && isKeywordField(mappedFieldType)) {
164-
fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText)
165-
.caseInsensitive(kqlParserExecutionContext.isCaseSensitive() == false);
185+
fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
166186
} else if (hasWildcard) {
167187
fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName);
168188
} else if (isDateField(mappedFieldType)) {
169-
// TODO: add timezone
170-
fieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText);
189+
RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText);
190+
if (kqlParsingContext.timeZone() != null) {
191+
rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId());
192+
}
193+
fieldQuery = rangeFieldQuery;
171194
} else if (isKeywordField(mappedFieldType)) {
172-
fieldQuery = QueryBuilders.termQuery(fieldName, queryText)
173-
.caseInsensitive(kqlParserExecutionContext.isCaseSensitive() == false);
195+
fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive());
174196
} else if (ctx.fieldQueryValue().QUOTED_STRING() != null) {
175197
fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText);
176198
} else {
@@ -194,7 +216,26 @@ private static boolean isOrQuery(KqlBaseParser.BooleanQueryContext ctx) {
194216
}
195217

196218
private void withFields(KqlBaseParser.FieldNameContext ctx, BiConsumer<String, MappedFieldType> fieldConsummer) {
197-
kqlParserExecutionContext.resolveFields(ctx).forEach(fieldDef -> fieldConsummer.accept(fieldDef.v1(), fieldDef.v2()));
219+
assert ctx != null : "Field ctx cannot be null";
220+
String fieldNamePattern = extractText(ctx);
221+
Set<String> fieldNames = kqlParsingContext.resolveFieldNames(fieldNamePattern);
222+
223+
if (ctx.value.getType() == KqlBaseParser.QUOTED_STRING && Regex.isSimpleMatchPattern(fieldNamePattern)) {
224+
// When using quoted string, wildcards are not expanded.
225+
// No field can match and we can return early.
226+
return;
227+
}
228+
229+
if (ctx.value.getType() == KqlBaseParser.QUOTED_STRING) {
230+
assert fieldNames.size() < 2 : "expecting only one matching field";
231+
}
232+
233+
fieldNames.forEach(fieldName -> {
234+
MappedFieldType fieldType = kqlParsingContext.fieldType(fieldName);
235+
if (isSearchableField(fieldName, fieldType)) {
236+
fieldConsummer.accept(fieldName, fieldType);
237+
}
238+
});
198239
}
199240

200241
private QueryBuilder rewriteDisjunctionQuery(BoolQueryBuilder boolQueryBuilder) {

0 commit comments

Comments
 (0)