diff --git a/docs/changelog/135895.yaml b/docs/changelog/135895.yaml new file mode 100644 index 0000000000000..a8f660ee815c9 --- /dev/null +++ b/docs/changelog/135895.yaml @@ -0,0 +1,6 @@ +pr: 135895 +summary: Add optional parameters support to KQL function +area: ES|QL +type: enhancement +issues: + - 135823 diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md index 862d8f050e87e..2d0ad3d84c2d5 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/kql.md @@ -1,6 +1,8 @@ % This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. -**Example** +**Examples** + +Use KQL to filter by a specific field value ```esql FROM books @@ -15,4 +17,15 @@ FROM books | 2883 | William Faulkner | | 3293 | Danny Faulkner | +```{applies_to} +stack: ga 9.3.0 +``` + +Use KQL with additional options for case-insensitive matching and custom settings + +```esql +FROM employees +| WHERE KQL("mary", {"case_insensitive": true, "default_field": "first_name", "boost": 1.5}) +``` + diff --git a/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md new file mode 100644 index 0000000000000..b0dbba48647b2 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/kql.md @@ -0,0 +1,16 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported function named parameters** + +`boost` +: (float) Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0. + +`time_zone` +: (keyword) UTC offset or IANA time zone used to interpret date literals in the query string. + +`case_insensitive` +: (boolean) If true, performs case-insensitive matching for keyword fields. Defaults to false. + +`default_field` +: (keyword) Default field to search if no field is provided in the query string. Supports wildcards (*). + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/kql.md b/docs/reference/query-languages/esql/_snippets/functions/layout/kql.md index 94a156bdc8a8a..20515c96854ee 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/layout/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/kql.md @@ -22,5 +22,8 @@ stack: preview 9.0.0, ga 9.1.0 :::{include} ../types/kql.md ::: +:::{include} ../functionNamedParams/kql.md +::: + :::{include} ../examples/kql.md ::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md index 7532069aa5ed2..9ffaec37b1f56 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/kql.md @@ -5,3 +5,6 @@ `query` : Query string in KQL query string format. +`options` +: (Optional) KQL additional options as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). Available in stack version 9.3.0 and later. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/kql.md b/docs/reference/query-languages/esql/_snippets/functions/types/kql.md index 0af3d49fd7399..fca2723a34442 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/kql.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/kql.md @@ -2,8 +2,10 @@ **Supported types** -| query | result | -| --- | --- | -| keyword | boolean | -| text | boolean | +| query | options | result | +| --- | --- | --- | +| keyword | named parameters | boolean | +| keyword | | boolean | +| text | named parameters | boolean | +| text | | boolean | diff --git a/docs/reference/query-languages/esql/images/functions/kql.svg b/docs/reference/query-languages/esql/images/functions/kql.svg index 0700b1bf2ce1c..3b860bd88786b 100644 --- a/docs/reference/query-languages/esql/images/functions/kql.svg +++ b/docs/reference/query-languages/esql/images/functions/kql.svg @@ -1 +1 @@ -KQL(query) \ No newline at end of file +KQL(query,options) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json index f3fe7c2df4c11..761e462f1b53e 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/kql.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/kql.json @@ -16,6 +16,25 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Query string in KQL query string format." + }, + { + "name" : "options", + "type" : "function_named_parameters", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field to search if no field is provided in the query string. Supports wildcards (*).'}", + "optional" : true, + "description" : "(Optional) KQL additional options as <>. Available in stack version 9.3.0 and later." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { @@ -27,10 +46,30 @@ ], "variadic" : false, "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Query string in KQL query string format." + }, + { + "name" : "options", + "type" : "function_named_parameters", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='time_zone', values=[UTC, Europe/Paris, America/New_York], description='UTC offset or IANA time zone used to interpret date literals in the query string.'}, {name='case_insensitive', values=[true, false], description='If true, performs case-insensitive matching for keyword fields. Defaults to false.'}, {name='default_field', values=[*, logs.*, title], description='Default field to search if no field is provided in the query string. Supports wildcards (*).'}", + "optional" : true, + "description" : "(Optional) KQL additional options as <>. Available in stack version 9.3.0 and later." + } + ], + "variadic" : false, + "returnType" : "boolean" } ], "examples" : [ - "FROM books\n| WHERE KQL(\"author: Faulkner\")" + "FROM books\n| WHERE KQL(\"author: Faulkner\")", + "FROM employees\n| WHERE KQL(\"mary\", {\"case_insensitive\": true, \"default_field\": \"first_name\", \"boost\": 1.5})" ], "preview" : false, "snapshot_only" : false diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec index 8f7d3446c899c..204d17f57ab50 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec @@ -287,3 +287,106 @@ c: long | scalerank: long 10 | 3 15 | 2 ; + +kqlWithOptions +required_capability: kql_function +required_capability: kql_function_options + + +FROM employees +| WHERE KQL("first_name: Mary", {"case_insensitive": false}) +| KEEP emp_no, first_name, last_name +; + +emp_no:integer | first_name:keyword | last_name:keyword +10011 | Mary | Sluis +; + +kqlWithCaseInsensitiveOption +required_capability: kql_function +required_capability: kql_function_options + +FROM employees +| WHERE KQL("first_name: mary", {"case_insensitive": true}) +| KEEP emp_no, first_name, last_name +; + +emp_no:integer | first_name:keyword | last_name:keyword +10011 | Mary | Sluis +; + +kqlWithTimeZoneOption +required_capability: kql_function +required_capability: kql_function_options + +FROM logs +| WHERE KQL("@timestamp > \"2023-10-23T09:56:00\" AND @timestamp < \"2023-10-23T09:57:00\"", {"time_zone": "America/New_York"}) +| KEEP @timestamp, message +| SORT @timestamp ASC +; + +@timestamp:date | message:text +2023-10-23T13:56:01.543Z | No response +2023-10-23T13:56:01.544Z | Running cats (cycle 2) +; + +kqlWithDefaultFieldOption +required_capability: kql_function +required_capability: kql_function_options + +FROM employees +| WHERE KQL("Support Engineer", {"default_field": "job_positions"}) +| KEEP emp_no, first_name, last_name, job_positions +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword +10004 | Chirstian | Koblick | [Head Human Resources, Reporting Analyst, Support Engineer, Tech Lead] +10015 | Guoxiang | Nooteboom | [Head Human Resources, Junior Developer, Principal Support Engineer, Support Engineer] +10021 | Ramzi | Erde | Support Engineer +; + +kqlWithBoostOption +required_capability: kql_function +required_capability: kql_function_options + +FROM employees METADATA _score +| WHERE KQL("job_positions: Support Engineer", {"boost": 2.5}) +| KEEP emp_no, first_name, last_name, job_positions +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword +10004 | Chirstian | Koblick | [Head Human Resources, Reporting Analyst, Support Engineer, Tech Lead] +10015 | Guoxiang | Nooteboom | [Head Human Resources, Junior Developer, Principal Support Engineer, Support Engineer] +10021 | Ramzi | Erde | Support Engineer +; + +kqlWithMultipleOptions +required_capability: kql_function +required_capability: kql_function_options +// tag::kql-with-options[] +FROM employees +| WHERE KQL("mary", {"case_insensitive": true, "default_field": "first_name", "boost": 1.5}) +// end::kql-with-options[] +| KEEP emp_no, first_name, last_name +; + +emp_no:integer | first_name:keyword | last_name:keyword +10011 | Mary | Sluis +; + +kqlWithWildcardDefaultField +required_capability: kql_function +required_capability: kql_function_options + +FROM employees +| WHERE KQL("Mary", {"default_field": "*_name"}) +| KEEP emp_no, first_name, last_name +; + +emp_no:integer | first_name:keyword | last_name:keyword +10011 | Mary | Sluis +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 3cfd1deea10d3..6ab690cafe9c5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -734,6 +734,11 @@ public enum Cap { */ KQL_FUNCTION, + /** + * Support for optional parameters in KQL function (case_insensitive, time_zone, default_field, boost). + */ + KQL_FUNCTION_OPTIONS, + /** * Hash function */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 077c9aaa2b7b6..8553b3f587372 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -504,7 +504,7 @@ private static FunctionDefinition[][] functions() { // fulltext functions new FunctionDefinition[] { def(Decay.class, quad(Decay::new), "decay"), - def(Kql.class, uni(Kql::new), "kql"), + def(Kql.class, bi(Kql::new), "kql"), def(Knn.class, tri(Knn::new), "knn"), def(Match.class, tri(Match::new), "match"), def(MultiMatch.class, MultiMatch::new, "multi_match"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java index a9eb30ab3e52e..3f0eac393e7fe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -12,15 +12,21 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.MapExpression; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.Foldables; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.MapParam; +import org.elasticsearch.xpack.esql.expression.function.OptionalArgument; +import org.elasticsearch.xpack.esql.expression.function.Options; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; @@ -28,21 +34,57 @@ import org.elasticsearch.xpack.esql.querydsl.query.KqlQuery; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Map.entry; +import static org.elasticsearch.index.query.AbstractQueryBuilder.BOOST_FIELD; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNull; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; +import static org.elasticsearch.xpack.esql.expression.Foldables.TypeResolutionValidator.forPreOptimizationValidation; +import static org.elasticsearch.xpack.esql.expression.Foldables.resolveTypeQuery; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.CASE_INSENSITIVE_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.DEFAULT_FIELD_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.TIME_ZONE_FIELD; /** * Full text function that performs a {@link KqlQuery} . */ -public class Kql extends FullTextFunction { +public class Kql extends FullTextFunction implements OptionalArgument { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Kql", Kql::readFrom); + // Options for KQL function. They don't need to be serialized as the data nodes will retrieve them from the query builder + private final transient Expression options; + + public static final Map ALLOWED_OPTIONS = Map.ofEntries( + entry(BOOST_FIELD.getPreferredName(), FLOAT), + entry(CASE_INSENSITIVE_FIELD.getPreferredName(), BOOLEAN), + entry(TIME_ZONE_FIELD.getPreferredName(), KEYWORD), + entry(DEFAULT_FIELD_FIELD.getPreferredName(), KEYWORD) + ); + @FunctionInfo( returnType = "boolean", appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.0.0"), @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.GA, version = "9.1.0") }, description = "Performs a KQL query. Returns true if the provided KQL query string matches the row.", - examples = { @Example(file = "kql-function", tag = "kql-with-field") } + examples = { + @Example(file = "kql-function", tag = "kql-with-field", description = "Use KQL to filter by a specific field value"), + @Example( + file = "kql-function", + tag = "kql-with-options", + description = "Use KQL with additional options for case-insensitive matching and custom settings", + applies_to = "stack: ga 9.3.0" + ) } ) public Kql( Source source, @@ -50,13 +92,45 @@ public Kql( name = "query", type = { "keyword", "text" }, description = "Query string in KQL query string format." - ) Expression queryString + ) Expression queryString, + @MapParam( + name = "options", + description = "(Optional) KQL additional options as <>." + + " Available in stack version 9.3.0 and later.", + params = { + @MapParam.MapParamEntry( + name = "case_insensitive", + type = "boolean", + valueHint = { "true", "false" }, + description = "If true, performs case-insensitive matching for keyword fields. Defaults to false." + ), + @MapParam.MapParamEntry( + name = "time_zone", + type = "keyword", + valueHint = { "UTC", "Europe/Paris", "America/New_York" }, + description = "UTC offset or IANA time zone used to interpret date literals in the query string." + ), + @MapParam.MapParamEntry( + name = "default_field", + type = "keyword", + valueHint = { "*", "logs.*", "title" }, + description = "Default field to search if no field is provided in the query string. Supports wildcards (*)." + ), + @MapParam.MapParamEntry( + name = "boost", + type = "float", + valueHint = { "2.5" }, + description = "Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0." + ) }, + optional = true + ) Expression options ) { - super(source, queryString, List.of(queryString), null); + this(source, queryString, options, null); } - public Kql(Source source, Expression queryString, QueryBuilder queryBuilder) { - super(source, queryString, List.of(queryString), queryBuilder); + public Kql(Source source, Expression queryString, Expression options, QueryBuilder queryBuilder) { + super(source, queryString, options == null ? List.of(queryString) : List.of(queryString, options), queryBuilder); + this.options = options; } private static Kql readFrom(StreamInput in) throws IOException { @@ -66,7 +140,8 @@ private static Kql readFrom(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); } - return new Kql(source, query, queryBuilder); + // Options are not serialized - they're embedded in the QueryBuilder + return new Kql(source, query, null, queryBuilder); } @Override @@ -76,6 +151,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { out.writeOptionalNamedWriteable(queryBuilder()); } + // Options are not serialized - they're embedded in the QueryBuilder } @Override @@ -83,23 +159,75 @@ public String getWriteableName() { return ENTRY.name; } + public Expression options() { + return options; + } + + private TypeResolution resolveQuery() { + TypeResolution result = isType(query(), t -> t == KEYWORD || t == TEXT, sourceText(), FIRST, "keyword, text").and( + isNotNull(query(), sourceText(), FIRST) + ); + if (result.unresolved()) { + return result; + } + result = resolveTypeQuery(query(), sourceText(), forPreOptimizationValidation(query())); + if (result.equals(TypeResolution.TYPE_RESOLVED) == false) { + return result; + } + return TypeResolution.TYPE_RESOLVED; + } + + @Override + protected TypeResolution resolveParams() { + return resolveQuery().and(Options.resolve(options(), source(), SECOND, ALLOWED_OPTIONS)); + } + + private Map kqlQueryOptions() throws InvalidArgumentException { + if (options() == null) { + return null; + } + + Map kqlOptions = new HashMap<>(); + Options.populateMap((MapExpression) options(), kqlOptions, source(), SECOND, ALLOWED_OPTIONS); + return kqlOptions; + } + @Override public Expression replaceChildren(List newChildren) { - return new Kql(source(), newChildren.get(0), queryBuilder()); + return new Kql(source(), newChildren.get(0), newChildren.size() > 1 ? newChildren.get(1) : null, queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, Kql::new, query(), queryBuilder()); + return NodeInfo.create(this, Kql::new, query(), options(), queryBuilder()); } @Override protected Query translate(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { - return new KqlQuery(source(), Foldables.queryAsString(query(), sourceText())); + return new KqlQuery(source(), Foldables.queryAsString(query(), sourceText()), kqlQueryOptions()); } @Override public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { - return new Kql(source(), query(), queryBuilder); + return new Kql(source(), query(), options(), queryBuilder); + } + + @Override + public boolean equals(Object o) { + // KQL does not serialize options, as they get included in the query builder. We need to override equals and hashcode to + // ignore options when comparing. + if (o == null || getClass() != o.getClass()) return false; + var kql = (Kql) o; + return Objects.equals(query(), kql.query()) && Objects.equals(queryBuilder(), kql.queryBuilder()); + } + + @Override + public int hashCode() { + return Objects.hash(query(), queryBuilder()); + } + + @Override + public String toString() { + return "Kql{" + "query=" + query() + (options == null ? "" : ", options=" + options) + '}'; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java index 56229ec325d73..07e8d370c876a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.esql.querydsl.query; -import org.elasticsearch.core.Booleans; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -18,28 +17,27 @@ import java.util.function.BiConsumer; import static java.util.Map.entry; +import static org.elasticsearch.index.query.AbstractQueryBuilder.BOOST_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.CASE_INSENSITIVE_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.DEFAULT_FIELD_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.TIME_ZONE_FIELD; public class KqlQuery extends Query { - private static final Map> BUILDER_APPLIERS = Map.ofEntries( - entry(KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), KqlQueryBuilder::timeZone), - entry(KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), KqlQueryBuilder::defaultField), - entry(KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), (qb, s) -> qb.caseInsensitive(Booleans.parseBoolean(s))) + private static final Map> BUILDER_APPLIERS = Map.ofEntries( + entry(TIME_ZONE_FIELD.getPreferredName(), (qb, v) -> qb.timeZone((String) v)), + entry(DEFAULT_FIELD_FIELD.getPreferredName(), (qb, v) -> qb.defaultField((String) v)), + entry(CASE_INSENSITIVE_FIELD.getPreferredName(), (qb, v) -> qb.caseInsensitive((Boolean) v)), + entry(BOOST_FIELD.getPreferredName(), (qb, v) -> qb.boost(((Number) v).floatValue())) ); private final String query; + private final Map options; - private final Map options; - - // dedicated constructor for QueryTranslator - public KqlQuery(Source source, String query) { - this(source, query, null); - } - - public KqlQuery(Source source, String query, Map options) { + public KqlQuery(Source source, String query, Map options) { super(source); this.query = query; - this.options = options == null ? Collections.emptyMap() : options; + this.options = options == null ? Collections.emptyMap() : Map.copyOf(options); } @Override @@ -55,27 +53,22 @@ protected QueryBuilder asBuilder() { return queryBuilder; } + @Override + public boolean containsPlan() { + return false; + } + public String query() { return query; } - public Map options() { + public Map options() { return options; } @Override - public int hashCode() { - return Objects.hash(query, options); - } - - @Override - public boolean equals(Object obj) { - if (false == super.equals(obj)) { - return false; - } - - KqlQuery other = (KqlQuery) obj; - return Objects.equals(query, other.query) && Objects.equals(options, other.options); + public boolean scorable() { + return true; } @Override @@ -84,12 +77,16 @@ protected String innerToString() { } @Override - public boolean scorable() { - return true; + public int hashCode() { + return Objects.hash(query, options); } @Override - public boolean containsPlan() { - return false; + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + KqlQuery other = (KqlQuery) obj; + return Objects.equals(query, other.query) && Objects.equals(options, other.options); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 15198ff179f3e..4aeeb693779d7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchPhrase; import org.elasticsearch.xpack.esql.expression.function.fulltext.MultiMatch; @@ -2295,6 +2296,9 @@ public void testFullTextFunctionOptions() { if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", title, body, {\"%s\": %s})"); } + if (EsqlCapabilities.Cap.KQL_FUNCTION_OPTIONS.isEnabled()) { + checkOptionDataTypes(Kql.ALLOWED_OPTIONS, "FROM test | WHERE KQL(\"title: Jean\", {\"%s\": %s})"); + } } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlErrorTests.java index 891c419841e70..8377ce664a345 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlErrorTests.java @@ -8,16 +8,21 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.hamcrest.Matcher; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.stream.Stream; +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.hamcrest.Matchers.equalTo; public class KqlErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @@ -34,11 +39,42 @@ protected Stream> testCandidates(List cases, Se @Override protected Expression build(Source source, List args) { - return new Kql(source, args.get(0)); + return new Kql(source, args.getFirst(), args.size() > 1 ? args.get(1) : null); } @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + return equalTo(errorMessageStringForKql(validPerPosition, signature, (l, p) -> "keyword, text")); + } + + private static String errorMessageStringForKql( + List> validPerPosition, + List signature, + AbstractFunctionTestCase.PositionalErrorMessageSupplier positionalErrorMessageSupplier + ) { + boolean invalid = false; + for (int i = 0; i < signature.size() && invalid == false; i++) { + // Need to check for nulls and bad parameters in order + if (signature.get(i) == DataType.NULL) { + return TypeResolutions.ParamOrdinal.fromIndex(i).name().toLowerCase(Locale.ROOT) + + " argument of [" + + sourceForSignature(signature) + + "] cannot be null, received []"; + } + if (validPerPosition.get(i).contains(signature.get(i)) == false) { + // Map expressions have different error messages + if (i == 1) { + return format(null, "second argument of [{}] must be a map expression, received []", sourceForSignature(signature)); + } + break; + } + } + + try { + return typeErrorMessage(true, validPerPosition, signature, positionalErrorMessageSupplier); + } catch (IllegalStateException e) { + // This means all the positional args were okay, so the expected error is for nulls or from the combination + return EsqlBinaryComparison.formatIncompatibleTypesMessage(signature.get(0), signature.get(1), sourceForSignature(signature)); + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java index 118bf56c5d6b9..1b34aa9fefe31 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java @@ -10,13 +10,26 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.MapExpression; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; +import static org.elasticsearch.xpack.esql.SerializationTestUtils.serializeDeserialize; +import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; +import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER; +import static org.hamcrest.Matchers.equalTo; + public class KqlTests extends NoneFieldFullTextFunctionTestCase { public KqlTests(@Name("TestCase") Supplier testCaseSupplier) { super(testCaseSupplier); @@ -24,11 +37,62 @@ public KqlTests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { - return generateParameters(); + return parameterSuppliersFromTypedData(addFunctionNamedParams(getStringTestSupplier())); + } + + /** + * Adds function named parameters to all the test case suppliers provided + */ + private static List addFunctionNamedParams(List suppliers) { + List result = new ArrayList<>(suppliers); + for (TestCaseSupplier supplier : suppliers) { + List dataTypes = new ArrayList<>(supplier.types()); + dataTypes.add(UNSUPPORTED); + result.add(new TestCaseSupplier(supplier.name() + ", options", dataTypes, () -> { + List values = new ArrayList<>(supplier.get().getData()); + values.add( + new TestCaseSupplier.TypedData( + new MapExpression(Source.EMPTY, List.of(Literal.keyword(Source.EMPTY, "case_insensitive"), Literal.TRUE)), + UNSUPPORTED, + "options" + ).forceLiteral() + ); + + return new TestCaseSupplier.TestCase(values, equalTo("KqlEvaluator"), BOOLEAN, equalTo(true)); + })); + } + return result; } @Override protected Expression build(Source source, List args) { - return new Kql(source, args.get(0)); + Kql kql = new Kql(source, args.get(0), args.size() > 1 ? args.get(1) : null); + // We need to add the QueryBuilder to the kql expression, as it is used to implement equals() and hashCode() and + // thus test the serialization methods. But we can only do this if the parameters make sense. + if (args.get(0).foldable()) { + QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, kql).toQueryBuilder(); + kql = (Kql) kql.replaceQueryBuilder(queryBuilder); + } + return kql; + } + + /** + * Copy of the overridden method that doesn't check for children size, as the {@code options} child isn't serialized in Kql. + */ + @Override + protected Expression serializeDeserializeExpression(Expression expression) { + Expression newExpression = serializeDeserialize( + expression, + PlanStreamOutput::writeNamedWriteable, + in -> in.readNamedWriteable(Expression.class), + testCase.getConfiguration() // The configuration query should be == to the source text of the function for this to work + ); + // Fields use synthetic sources, which can't be serialized. So we use the originals instead. + return newExpression.replaceChildren(expression.children()); + } + + @Override + public void testFold() { + // kql query cannot be folded. } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java index 8dfb50f84ac1e..fd82c11b60085 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.esql.querydsl.query; +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.SourceTests; @@ -14,6 +16,7 @@ import java.time.ZoneId; import java.time.zone.ZoneRulesException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -23,23 +26,31 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.elasticsearch.index.query.AbstractQueryBuilder.BOOST_FIELD; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.CASE_INSENSITIVE_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.DEFAULT_FIELD_FIELD; +import static org.elasticsearch.xpack.kql.query.KqlQueryBuilder.TIME_ZONE_FIELD; import static org.hamcrest.Matchers.equalTo; public class KqlQueryTests extends ESTestCase { static KqlQuery randomKqkQueryQuery() { - Map options = new HashMap<>(); + Map options = new HashMap<>(); if (randomBoolean()) { - options.put(KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), String.valueOf(randomBoolean())); + options.put(CASE_INSENSITIVE_FIELD.getPreferredName(), randomBoolean()); } if (randomBoolean()) { - options.put(KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), randomIdentifier()); + options.put(DEFAULT_FIELD_FIELD.getPreferredName(), randomIdentifier()); } if (randomBoolean()) { - options.put(KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), randomZone().getId()); + options.put(TIME_ZONE_FIELD.getPreferredName(), randomZone().getId()); + } + + if (randomBoolean()) { + options.put(BOOST_FIELD.getPreferredName(), randomFloat() * 5.0f + 0.1f); // Random float between 0.1 and 5.1 } return new KqlQuery(SourceTests.randomSource(), randomAlphaOfLength(5), Collections.unmodifiableMap(options)); @@ -65,8 +76,8 @@ private static KqlQuery mutate(KqlQuery query) { return randomFrom(options).apply(query); } - private static Map mutateOptions(Map options) { - Map mutatedOptions = new HashMap<>(options); + private static Map mutateOptions(Map options) { + Map mutatedOptions = new HashMap<>(options); if (options.isEmpty() == false && randomBoolean()) { mutatedOptions = options.entrySet() .stream() @@ -76,48 +87,48 @@ private static Map mutateOptions(Map options) { while (mutatedOptions.equals(options)) { if (randomBoolean()) { - mutatedOptions = mutateOption( - mutatedOptions, - KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), - () -> String.valueOf(randomBoolean()) - ); + mutatedOptions = mutateOption(mutatedOptions, CASE_INSENSITIVE_FIELD.getPreferredName(), () -> randomBoolean()); + } + + if (randomBoolean()) { + mutatedOptions = mutateOption(mutatedOptions, DEFAULT_FIELD_FIELD.getPreferredName(), () -> randomIdentifier()); } if (randomBoolean()) { - mutatedOptions = mutateOption( - mutatedOptions, - KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), - () -> randomIdentifier() - ); + mutatedOptions = mutateOption(mutatedOptions, TIME_ZONE_FIELD.getPreferredName(), () -> randomZone().getId()); } if (randomBoolean()) { - mutatedOptions = mutateOption( - mutatedOptions, - KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), - () -> randomZone().getId() - ); + mutatedOptions = mutateOption(mutatedOptions, "boost", () -> randomFloat() * 5.0f + 0.1f); } } return Collections.unmodifiableMap(mutatedOptions); } - private static Map mutateOption(Map options, String optionName, Supplier valueSupplier) { + private static Map mutateOption(Map options, String optionName, Supplier valueSupplier) { options = new HashMap<>(options); options.put(optionName, randomValueOtherThan(options.get(optionName), valueSupplier)); return options; } public void testQueryBuilding() { - KqlQueryBuilder qb = getBuilder(Map.of("case_insensitive", "false")); + KqlQueryBuilder qb = getBuilder(Map.of("case_insensitive", false)); assertThat(qb.caseInsensitive(), equalTo(false)); - qb = getBuilder(Map.of("case_insensitive", "false", "time_zone", "UTC", "default_field", "foo")); + qb = getBuilder(Map.of("case_insensitive", false, "time_zone", "UTC", "default_field", "foo")); assertThat(qb.caseInsensitive(), equalTo(false)); assertThat(qb.timeZone(), equalTo(ZoneId.of("UTC"))); assertThat(qb.defaultField(), equalTo("foo")); + qb = getBuilder(Map.of("boost", 2.5f)); + assertThat(qb.boost(), equalTo(2.5f)); + + qb = getBuilder(Map.of("case_insensitive", true, "boost", 1.5f, "default_field", "content")); + assertThat(qb.caseInsensitive(), equalTo(true)); + assertThat(qb.boost(), equalTo(1.5f)); + assertThat(qb.defaultField(), equalTo("content")); + Exception e = expectThrows(IllegalArgumentException.class, () -> getBuilder(Map.of("pizza", "yummy"))); assertThat(e.getMessage(), equalTo("illegal kql query option [pizza]")); @@ -125,7 +136,7 @@ public void testQueryBuilding() { assertThat(e.getMessage(), equalTo("Unknown time-zone ID: aoeu")); } - private static KqlQueryBuilder getBuilder(Map options) { + private static KqlQueryBuilder getBuilder(Map options) { final Source source = new Source(1, 1, StringUtils.EMPTY); final KqlQuery kqlQuery = new KqlQuery(source, "eggplant", options); return (KqlQueryBuilder) kqlQuery.asBuilder(); @@ -136,4 +147,11 @@ public void testToString() { final KqlQuery kqlQuery = new KqlQuery(source, "eggplant", Map.of()); assertEquals("KqlQuery@1:2[eggplant]", kqlQuery.toString()); } + + @Override + protected NamedWriteableRegistry writableRegistry() { + List entries = new ArrayList<>(ClusterModule.getNamedWriteables()); + entries.add(new NamedWriteableRegistry.Entry(KqlQueryBuilder.class, KqlQueryBuilder.NAME, KqlQueryBuilder::new)); + return new NamedWriteableRegistry(entries); + } } diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java index 9d7aeb4bf0ddf..64e158c6c9261 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java @@ -26,6 +26,8 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isDateField; @@ -181,15 +183,32 @@ public QueryBuilder visitRangeQuery(KqlBaseParser.RangeQueryContext ctx) { @Override public QueryBuilder visitFieldLessQuery(KqlBaseParser.FieldLessQueryContext ctx) { String queryText = extractText(ctx.fieldQueryValue()); + boolean hasWildcard = hasWildcard(ctx.fieldQueryValue()); + boolean isPhraseMatch = ctx.fieldQueryValue().fieldQueryValueLiteral().QUOTED_STRING() != null; + + if (kqlParsingContext.caseInsensitive() && isPhraseMatch == false) { + // Special handling for case-insenitive queries. + // We can't use a query_string or a match query since it does not support case-insensitive on all field types. + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + Set searchFields = kqlParsingContext.resolveDefaultFieldNames() + .stream() + .filter(kqlParsingContext::isSearchableField) + .filter(Predicate.not(kqlParsingContext::isDateField)) // Date fields do not support case insensitive + .filter(Predicate.not(kqlParsingContext::isRangeField)) // Range fields do not support case insensitive + .collect(Collectors.toSet()); + + withFields(searchFields, fieldQueryApplier(queryText, hasWildcard, false, boolQueryBuilder::should)); + return rewriteDisjunctionQuery(boolQueryBuilder); + } - if (hasWildcard(ctx.fieldQueryValue())) { + if (hasWildcard) { + // Wildcard queries are using a query_string query QueryStringQueryBuilder queryString = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)); if (kqlParsingContext.defaultField() != null) { queryString.defaultField(kqlParsingContext.defaultField()); } return queryString; } - boolean isPhraseMatch = ctx.fieldQueryValue().fieldQueryValueLiteral().QUOTED_STRING() != null; MultiMatchQueryBuilder multiMatchQuery = QueryBuilders.multiMatchQuery(queryText) .type(isPhraseMatch ? MultiMatchQueryBuilder.Type.PHRASE : MultiMatchQueryBuilder.Type.BEST_FIELDS) @@ -250,37 +269,47 @@ public QueryBuilder parseFieldQuery( BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); String queryText = extractText(fieldQueryValueCtx.fieldQueryValueLiteral()); boolean hasWildcard = hasWildcard(fieldQueryValueCtx); + boolean isPhraseMatch = fieldQueryValueCtx.fieldQueryValueLiteral().QUOTED_STRING() != null; - withFields(fieldNameCtx, (fieldName, mappedFieldType) -> { - QueryBuilder fieldQuery; - - if (hasWildcard && isKeywordField(mappedFieldType)) { - fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); - } else if (hasWildcard) { - fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName); - } else if (isDateField(mappedFieldType)) { - RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText); - if (kqlParsingContext.timeZone() != null) { - rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId()); - } - fieldQuery = rangeFieldQuery; - } else if (isKeywordField(mappedFieldType)) { - fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); - } else if (fieldQueryValueCtx.fieldQueryValueLiteral().QUOTED_STRING() != null) { - fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText); - } else { - fieldQuery = QueryBuilders.matchQuery(fieldName, queryText); - } - - if (fieldQuery != null) { - boolQueryBuilder.should(wrapWithNestedQuery(fieldName, fieldQuery)); - } - }); + withFields(fieldNameCtx, fieldQueryApplier(queryText, hasWildcard, isPhraseMatch, boolQueryBuilder::should)); return rewriteDisjunctionQuery(boolQueryBuilder); } } + private BiConsumer fieldQueryApplier( + String queryText, + boolean hasWildcard, + boolean isPhraseMatch, + Consumer clauseConsumer + ) { + return (fieldName, mappedFieldType) -> { + QueryBuilder fieldQuery; + + if (hasWildcard && isKeywordField(mappedFieldType)) { + fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); + } else if (hasWildcard) { + fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName); + } else if (isDateField(mappedFieldType)) { + RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText); + if (kqlParsingContext.timeZone() != null) { + rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId()); + } + fieldQuery = rangeFieldQuery; + } else if (isKeywordField(mappedFieldType)) { + fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); + } else if (isPhraseMatch) { + fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText); + } else { + fieldQuery = QueryBuilders.matchQuery(fieldName, queryText).lenient(true); + } + + if (fieldQuery != null) { + clauseConsumer.accept(wrapWithNestedQuery(fieldName, fieldQuery)); + } + }; + } + private static boolean isAndQuery(ParserRuleContext ctx) { return switch (ctx) { case KqlBaseParser.BooleanQueryContext booleanQueryCtx -> booleanQueryCtx.operator.getType() == KqlBaseParser.AND; @@ -311,6 +340,10 @@ private void withFields(KqlBaseParser.FieldNameContext ctx, BiConsumer fieldNames, BiConsumer fieldConsummer) { fieldNames.forEach(fieldName -> { MappedFieldType fieldType = kqlParsingContext.fieldType(fieldName); if (isSearchableField(fieldName, fieldType)) { diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java index 30740833ee40e..449ea740a5659 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java @@ -13,10 +13,12 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.NestedObjectMapper; +import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.support.NestedScope; import java.time.ZoneId; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -78,7 +80,22 @@ public Set resolveFieldNames(String fieldNamePattern) { } public Set resolveDefaultFieldNames() { - return resolveFieldNames(defaultField); + if (defaultField != null) { + return resolveFieldNames(defaultField); + } + + assert queryRewriteContext.getIndexSettings() != null; + + if (queryRewriteContext.getIndexSettings().getDefaultFields().isEmpty()) { + return resolveFieldNames("*"); + } + + Set fieldNames = new HashSet<>(); + queryRewriteContext.getIndexSettings().getDefaultFields().forEach(fieldNamePattern -> { + fieldNames.addAll(resolveFieldNames(fieldNamePattern)); + }); + + return fieldNames; } public MappedFieldType fieldType(String fieldName) { @@ -89,8 +106,17 @@ public static boolean isRuntimeField(MappedFieldType fieldType) { return fieldType instanceof AbstractScriptFieldType; } + public boolean isDateField(String fieldName) { + return isDateField(fieldType(fieldName)); + } + + public boolean isRangeField(String fieldName) { + return fieldType(fieldName) != null && fieldType(fieldName) instanceof RangeFieldMapper.RangeFieldType; + } + public static boolean isDateField(MappedFieldType fieldType) { - return fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE); + return fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE) + || fieldType.typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE); } public static boolean isKeywordField(MappedFieldType fieldType) { @@ -139,7 +165,7 @@ private Map nestedMappers() { public static class Builder { private final QueryRewriteContext queryRewriteContext; - private boolean caseInsensitive = true; + private boolean caseInsensitive = false; private ZoneId timeZone = null; private String defaultField = null; diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserFieldlessQueryTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserFieldlessQueryTests.java index c1f080fdc1eb4..86d5a407eb099 100644 --- a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserFieldlessQueryTests.java +++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/KqlParserFieldlessQueryTests.java @@ -7,7 +7,20 @@ package org.elasticsearch.xpack.kql.parser; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchNoneQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; public class KqlParserFieldlessQueryTests extends AbstractKqlParserTestCase { @@ -92,4 +105,182 @@ public void testParseQuotedStringQuery() { // Containing unescaped KQL reserved characters assertMultiMatchQuery(parseKqlQuery("\"foo*: {(})\""), "foo*: {(})", MultiMatchQueryBuilder.Type.PHRASE); } + + public void testParseCaseInsensitiveFieldlessQueries() { + // Test case-insensitive unquoted literal queries + QueryBuilder query = parseKqlQueryCaseInsensitive("Foo"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + assertThat(boolQuery.should(), hasSize(searchableFields().size() - excludedFieldCount())); + + // Verify that each should clause targets a searchable non-date field + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof MatchQueryBuilder matchQuery) { + assertThat(matchQuery.value(), equalTo("Foo")); + assertThat(matchQuery.lenient(), equalTo(true)); + } else if (clause instanceof TermQueryBuilder termQuery) { + assertThat(termQuery.value(), equalTo("Foo")); + assertThat(termQuery.caseInsensitive(), equalTo(true)); + } + } + } + + public void testParseCaseInsensitiveWildcardQueries() { + // Test case-insensitive wildcard queries + QueryBuilder query = parseKqlQueryCaseInsensitive("Fo*"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + assertThat(boolQuery.should(), hasSize(searchableFields().size() - excludedFieldCount())); + + // Verify that each should clause handles wildcards appropriately + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof WildcardQueryBuilder wildcardQuery) { + assertThat(wildcardQuery.value(), equalTo("Fo*")); + assertThat(wildcardQuery.caseInsensitive(), equalTo(true)); + } else if (clause instanceof QueryStringQueryBuilder queryStringQuery) { + assertThat(queryStringQuery.queryString(), equalTo("Fo*")); + } + } + } + + public void testParseCaseInsensitiveQuotedStringQueries() { + // Test case-insensitive phrase queries + // Note: With the current implementation, phrase matches use standard MultiMatchQuery + // even in case-insensitive mode (they bypass the special case-insensitive handling) + QueryBuilder query = parseKqlQueryCaseInsensitive("\"Foo Bar\""); + assertThat(query, instanceOf(MultiMatchQueryBuilder.class)); + + MultiMatchQueryBuilder multiMatchQuery = (MultiMatchQueryBuilder) query; + assertThat(multiMatchQuery.value(), equalTo("Foo Bar")); + assertThat(multiMatchQuery.type(), equalTo(MultiMatchQueryBuilder.Type.PHRASE)); + assertThat(multiMatchQuery.lenient(), equalTo(true)); + } + + public void testParseCaseInsensitiveMultipleWords() { + // Test case-insensitive multiple word queries + QueryBuilder query = parseKqlQueryCaseInsensitive("Foo Bar Baz"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + assertThat(boolQuery.should(), hasSize(searchableFields().size() - excludedFieldCount())); + + // Verify that each should clause is a match query with multiple words + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof MatchQueryBuilder matchQuery) { + assertThat(matchQuery.value(), equalTo("Foo Bar Baz")); + assertThat(matchQuery.lenient(), equalTo(true)); + } else if (clause instanceof TermQueryBuilder termQuery) { + assertThat(termQuery.value(), equalTo("Foo Bar Baz")); + assertThat(termQuery.caseInsensitive(), equalTo(true)); + } + } + } + + public void testParseCaseInsensitiveWildcardWithSpecialChars() { + // Test case-insensitive wildcard queries with special characters that need escaping + QueryBuilder query = parseKqlQueryCaseInsensitive("Fo*[bar]"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + assertThat(boolQuery.should(), hasSize(searchableFields().size() - excludedFieldCount())); + + // Verify proper escaping in query string queries + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof QueryStringQueryBuilder queryStringQuery) { + assertThat(queryStringQuery.queryString(), equalTo("Fo*\\[bar\\]")); + } + } + } + + public void testCaseInsensitiveWithEscapedCharacters() { + // Test case-insensitive queries with escaped characters + QueryBuilder query = parseKqlQueryCaseInsensitive("Foo\\*Bar"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + + // Should not be treated as wildcard since asterisk is escaped + List shouldClauses = boolQuery.should(); + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof MatchQueryBuilder matchQuery) { + assertThat(matchQuery.value(), equalTo("Foo*Bar")); + } else if (clause instanceof TermQueryBuilder termQuery) { + assertThat(termQuery.value(), equalTo("Foo*Bar")); + } + } + } + + public void testCaseInsensitiveWithCustomDefaultField() { + // Test case-insensitive queries with a custom default field pattern + QueryBuilder query = parseKqlQueryCaseInsensitiveWithDefaultField("foo", KEYWORD_FIELD_NAME); + + assertThat(query, instanceOf(TermQueryBuilder.class)); + TermQueryBuilder termQuery = (TermQueryBuilder) query; + assertThat(termQuery.fieldName(), equalTo(KEYWORD_FIELD_NAME)); + assertThat(termQuery.value(), equalTo("foo")); + assertThat(termQuery.caseInsensitive(), equalTo(true)); + } + + public void testCaseInsensitiveWithWildcardDefaultField() { + // Test case-insensitive queries with a wildcard default field pattern + QueryBuilder query = parseKqlQueryCaseInsensitiveWithDefaultField("foo", "mapped_*"); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + + // Should target all mapped fields that match the pattern (excluding date fields) + List expectedFields = searchableFields("mapped_*").stream() + .filter(fieldName -> fieldName.contains("date") == false && fieldName.contains("range") == false) + .toList(); + assertThat(boolQuery.should(), hasSize(expectedFields.size())); + + // Verify that all should clauses target appropriate fields + List shouldClauses = boolQuery.should(); // Verify that each should clause is properly configured + for (QueryBuilder clause : shouldClauses) { + if (clause instanceof MatchQueryBuilder matchQuery) { + assertThat(matchQuery.value(), equalTo("foo")); + assertThat(matchQuery.lenient(), equalTo(true)); + } else if (clause instanceof TermQueryBuilder termQuery) { + assertThat(termQuery.value(), equalTo("foo")); + assertThat(termQuery.caseInsensitive(), equalTo(true)); + } + } + } + + public void testCaseInsensitiveEmptyResultHandling() { + // Test behavior when no fields match after filtering (edge case) + // This creates a scenario where all default fields are date fields + QueryBuilder query = parseKqlQueryCaseInsensitiveWithDefaultField("test", DATE_FIELD_NAME); + assertThat(query, instanceOf(MatchNoneQueryBuilder.class)); + } + + /** + * Helper method to parse KQL query with case-insensitive mode enabled + */ + private QueryBuilder parseKqlQueryCaseInsensitive(String kqlQuery) { + KqlParser parser = new KqlParser(); + KqlParsingContext kqlParserContext = KqlParsingContext.builder(createQueryRewriteContext()).caseInsensitive(true).build(); + return parser.parseKqlQuery(kqlQuery, kqlParserContext); + } + + /** + * Helper method to parse KQL query with case-insensitive mode and custom default field + */ + private QueryBuilder parseKqlQueryCaseInsensitiveWithDefaultField(String kqlQuery, String defaultField) { + KqlParser parser = new KqlParser(); + KqlParsingContext kqlParserContext = KqlParsingContext.builder(createQueryRewriteContext()) + .caseInsensitive(true) + .defaultField(defaultField) + .build(); + return parser.parseKqlQuery(kqlQuery, kqlParserContext); + } + + /** + * Helper method to count the number of excluded fields in searchable fields. + * This is used to calculate expected should clause counts since date and range fields + * are filtered out in case-insensitive queries. + */ + private int excludedFieldCount() { + return (int) searchableFields().stream().filter(fieldName -> fieldName.contains("date") || fieldName.contains("range")).count(); + } } diff --git a/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml index 8255cd147d950..4f6e1d0ae64e5 100644 --- a/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml +++ b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml @@ -118,7 +118,6 @@ setup: --- "KQL match term queries (integer field)": - do: - catch: bad_request search: index: test-index rest_total_hits_as_int: true @@ -126,10 +125,7 @@ setup: { "query": { "kql": { "query": "integer_field: foo" } } } - - match: { error.type: "search_phase_execution_exception" } - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "failed to create query: For input string: \"foo\"" } - - contains: { error.root_cause.0.stack_trace: "Caused by: java.lang.NumberFormatException: For input string: \"foo\"" } + - match: { hits.total: 0 } - do: search: @@ -157,7 +153,6 @@ setup: --- "KQL match term queries (double field)": - do: - catch: bad_request search: index: test-index rest_total_hits_as_int: true @@ -165,10 +160,7 @@ setup: { "query": { "kql": { "query": "double_field: foo" } } } - - match: { error.type: "search_phase_execution_exception" } - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "failed to create query: For input string: \"foo\"" } - - contains: { error.root_cause.0.stack_trace: "Caused by: java.lang.NumberFormatException: For input string: \"foo\"" } + - match: { hits.total: 0 } - do: search: