diff --git a/docs/reference/esql/functions/description/kql.asciidoc b/docs/reference/esql/functions/description/kql.asciidoc new file mode 100644 index 0000000000000..e1fe411e6689c --- /dev/null +++ b/docs/reference/esql/functions/description/kql.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a KQL query. Returns true if the provided KQL query string matches the row. diff --git a/docs/reference/esql/functions/examples/kql.asciidoc b/docs/reference/esql/functions/examples/kql.asciidoc new file mode 100644 index 0000000000000..1f8518aeec394 --- /dev/null +++ b/docs/reference/esql/functions/examples/kql.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/kql-function.csv-spec[tag=kql-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/kql-function.csv-spec[tag=kql-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/kql.json b/docs/reference/esql/functions/kibana/definition/kql.json new file mode 100644 index 0000000000000..6960681fbbf0d --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/kql.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "kql", + "description" : "Performs a KQL query. Returns true if the provided KQL query string matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Query string in KQL query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Query string in KQL query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "FROM books \n| WHERE KQL(\"author: Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;" + ], + "preview" : true, + "snapshot_only" : true +} diff --git a/docs/reference/esql/functions/kibana/docs/kql.md b/docs/reference/esql/functions/kibana/docs/kql.md new file mode 100644 index 0000000000000..0ba419c1cd032 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/kql.md @@ -0,0 +1,14 @@ + + +### KQL +Performs a KQL query. Returns true if the provided KQL query string matches the row. + +``` +FROM books +| WHERE KQL("author: Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; +``` diff --git a/docs/reference/esql/functions/layout/kql.asciidoc b/docs/reference/esql/functions/layout/kql.asciidoc new file mode 100644 index 0000000000000..8cf2687b240c1 --- /dev/null +++ b/docs/reference/esql/functions/layout/kql.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-kql]] +=== `KQL` + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/kql.svg[Embedded,opts=inline] + +include::../parameters/kql.asciidoc[] +include::../description/kql.asciidoc[] +include::../types/kql.asciidoc[] +include::../examples/kql.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/kql.asciidoc b/docs/reference/esql/functions/parameters/kql.asciidoc new file mode 100644 index 0000000000000..6fb69323ff73c --- /dev/null +++ b/docs/reference/esql/functions/parameters/kql.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`query`:: +Query string in KQL query string format. diff --git a/docs/reference/esql/functions/signature/kql.svg b/docs/reference/esql/functions/signature/kql.svg new file mode 100644 index 0000000000000..3f550f27ccdff --- /dev/null +++ b/docs/reference/esql/functions/signature/kql.svg @@ -0,0 +1 @@ +KQL(query) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/kql.asciidoc b/docs/reference/esql/functions/types/kql.asciidoc new file mode 100644 index 0000000000000..866a39e925665 --- /dev/null +++ b/docs/reference/esql/functions/types/kql.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +query | result +keyword | boolean +text | boolean +|=== diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index f92c895cc5b7b..02f9752d21e09 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -34,6 +34,7 @@ dependencies { compileOnly project(':modules:lang-painless:spi') compileOnly project(xpackModule('esql-core')) compileOnly project(xpackModule('ml')) + implementation project(xpackModule('kql')) implementation project('compute') implementation project('compute:ann') implementation project(':libs:dissect') @@ -50,6 +51,7 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: xpackModule('enrich')) testImplementation project(path: xpackModule('spatial')) + testImplementation project(path: xpackModule('kql')) testImplementation project(path: ':modules:reindex') testImplementation project(path: ':modules:parent-join') diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec new file mode 100644 index 0000000000000..02be58efac774 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec @@ -0,0 +1,153 @@ +############################################### +# Tests for KQL function +# + +kqlWithField +required_capability: kql_function + +// tag::kql-with-field[] +FROM books +| WHERE KQL("author: Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; +// end::kql-with-field[] + +// tag::kql-with-field-result[] +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; +// end::kql-with-field-result[] + +kqlWithMultipleFields +required_capability: kql_function + +from books +| where kql("title:Return* AND author:*Tolkien") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +kqlWithQueryExpressions +required_capability: kql_function + +from books +| where kql(CONCAT("title:Return*", " AND author:*Tolkien")) +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +kqlWithConjunction +required_capability: kql_function + +from books +| where kql("title: Rings") and ratings > 4.6 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + +kqlWithFunctionPushedToLucene +required_capability: kql_function + +from hosts +| where kql("host: beta") and cidr_match(ip1, "127.0.0.2/32", "127.0.0.3/32") +| keep card, host, ip0, ip1; +ignoreOrder:true + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth1 |beta |127.0.0.1 |127.0.0.2 +; + +kqlWithNonPushableConjunction +required_capability: kql_function + +from books +| where kql("title: Rings") and length(title) > 75 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +; + +kqlWithMultipleWhereClauses +required_capability: kql_function + +from books +| where kql("title: rings") +| where kql("year > 1 AND year < 2005") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + + +kqlWithMultivaluedTextField +required_capability: kql_function + +from employees +| where kql("job_positions: Tech Lead AND job_positions:(Reporting Analyst)") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10004 | Chirstian | Koblick +10010 | Duangkaew | Piveteau +10011 | Mary | Sluis +10088 | Jungsoon | Syrzycki +10093 | Sailaja | Desikan +10097 | Remzi | Waschkowski +; + +kqlWithMultivaluedNumericField +required_capability: kql_function + +from employees +| where kql("salary_change > 14") +| keep emp_no, first_name, last_name, salary_change; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword | salary_change:double +10003 | Parto | Bamford | [12.82, 14.68] +10015 | Guoxiang | Nooteboom | [12.4, 14.25] +10023 | Bojan | Montemayor | [0.8, 14.63] +10040 | Weiyi | Meriste | [-8.94, 1.92, 6.97, 14.74] +10061 | Tse | Herber | [-2.58, -0.95, 14.39] +10065 | Satosi | Awdeh | [-9.81, -1.47, 14.44] +10099 | Valter | Sullins | [-8.78, -3.98, 10.71, 14.26] +; + +testMultiValuedFieldWithConjunction +required_capability: kql_function + +from employees +| where (kql("job_positions: (Data Scientist) OR job_positions:(Support Engineer)")) and gender == "F" +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10023 | Bojan | Montemayor +10041 | Uri | Lenart +10044 | Mingsen | Casley +10053 | Sanjiv | Zschoche +10069 | Margareta | Bierman +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec index 3e92e55928d64..6039dc05b6c44 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec @@ -101,8 +101,8 @@ book_no:keyword | title:text ; -matchMultivaluedTextField -required_capability: match_function +qstrWithMultivaluedTextField +required_capability: qstr_function from employees | where qstr("job_positions: (Tech Lead) AND job_positions:(Reporting Analyst)") @@ -118,8 +118,8 @@ emp_no:integer | first_name:keyword | last_name:keyword 10097 | Remzi | Waschkowski ; -matchMultivaluedNumericField -required_capability: match_function +qstrWithMultivaluedNumericField +required_capability: qstr_function from employees | where qstr("salary_change: [14 TO *]") @@ -137,7 +137,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | salary_change:double ; testMultiValuedFieldWithConjunction -required_capability: match_function +required_capability: qstr_function from employees | where (qstr("job_positions: (Data Scientist) OR job_positions:(Support Engineer)")) and gender == "F" diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java new file mode 100644 index 0000000000000..d58637ab52c86 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plugin; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.CoreMatchers.containsString; + +public class KqlFunctionIT extends AbstractEsqlIntegTestCase { + + @BeforeClass + protected static void ensureKqlFunctionEnabled() { + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + } + + @Before + public void setupIndex() { + createAndPopulateIndex(); + } + + public void testSimpleKqlQuery() { + var query = """ + FROM test + | WHERE kql("content: dog") + | KEEP id + | SORT id + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(1), List.of(3), List.of(4), List.of(5))); + } + } + + public void testMultiFieldKqlQuery() { + var query = """ + FROM test + | WHERE kql("dog OR canine") + | KEEP id + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValuesInAnyOrder(resp.values(), List.of(List.of(1), List.of(2), List.of(3), List.of(4), List.of(5))); + } + } + + public void testKqlQueryWithinEval() { + var query = """ + FROM test + | EVAL matches_query = kql("title: fox") + """; + + var error = expectThrows(VerificationException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE commands")); + } + + public void testInvalidKqlQueryEof() { + var query = """ + FROM test + | WHERE kql("content: ((((dog") + """; + + var error = expectThrows(QueryShardException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("Failed to parse KQL query [content: ((((dog]")); + assertThat(error.getRootCause().getMessage(), containsString("line 1:11: mismatched input '('")); + } + + public void testInvalidKqlQueryLexicalError() { + var query = """ + FROM test + | WHERE kql(":") + """; + + var error = expectThrows(QueryShardException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("Failed to parse KQL query [:]")); + assertThat(error.getRootCause().getMessage(), containsString("line 1:1: extraneous input ':' ")); + } + + private void createAndPopulateIndex() { + var indexName = "test"; + var client = client().admin().indices(); + var CreateRequest = client.prepareCreate(indexName) + .setSettings(Settings.builder().put("index.number_of_shards", 1)) + .setMapping("id", "type=integer", "content", "type=text"); + assertAcked(CreateRequest); + client().prepareBulk() + .add( + new IndexRequest(indexName).id("1") + .source("id", 1, "content", "The quick brown animal swiftly jumps over a lazy dog", "title", "A Swift Fox's Journey") + ) + .add( + new IndexRequest(indexName).id("2") + .source("id", 2, "content", "A speedy brown fox hops effortlessly over a sluggish canine", "title", "The Fox's Leap") + ) + .add( + new IndexRequest(indexName).id("3") + .source("id", 3, "content", "Quick and nimble, the fox vaults over the lazy dog", "title", "Brown Fox in Action") + ) + .add( + new IndexRequest(indexName).id("4") + .source( + "id", + 4, + "content", + "A fox that is quick and brown jumps over a dog that is quite lazy", + "title", + "Speedy Animals" + ) + ) + .add( + new IndexRequest(indexName).id("5") + .source( + "id", + 5, + "content", + "With agility, a quick brown fox bounds over a slow-moving dog", + "title", + "Foxes and Canines" + ) + ) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + ensureYellow(indexName); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index d675f772b5a3b..d9ce7fca312b3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -415,6 +415,11 @@ public enum Cap { */ MATCH_FUNCTION, + /** + * KQL function + */ + KQL_FUNCTION(Build.current().isSnapshot()), + /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. * https://github.com/elastic/elasticsearch/issues/112704 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 3ebb52641232e..2be13398dab2f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; @@ -793,14 +794,19 @@ private static void checkNotPresentInDisjunctions( private static void checkFullTextQueryFunctions(LogicalPlan plan, Set failures) { if (plan instanceof Filter f) { Expression condition = f.condition(); - checkCommandsBeforeExpression( - plan, - condition, - QueryString.class, - lp -> (lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation), - qsf -> "[" + qsf.functionName() + "] " + qsf.functionType(), - failures - ); + + List.of(QueryString.class, Kql.class).forEach(functionClass -> { + // Check for limitations of QSTR and KQL function. + checkCommandsBeforeExpression( + plan, + condition, + functionClass, + lp -> (lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation), + fullTextFunction -> "[" + fullTextFunction.functionName() + "] " + fullTextFunction.functionType(), + failures + ); + }); + checkCommandsBeforeExpression( plan, condition, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index ea1669ccc7a4f..3d26bc170b723 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Top; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.aggregate.WeightedAvg; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; @@ -411,6 +412,7 @@ private static FunctionDefinition[][] snapshotFunctions() { // This is an experimental function and can be removed without notice. def(Delay.class, Delay::new, "delay"), def(Categorize.class, Categorize::new, "categorize"), + def(Kql.class, Kql::new, "kql"), def(Rate.class, Rate::withUnresolvedTimestamp, "rate") } }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java index d59c736783172..8804a031de78c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java @@ -8,14 +8,28 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class FullTextWritables { public static List getNamedWriteables() { - return List.of(MatchQueryPredicate.ENTRY, MultiMatchQueryPredicate.ENTRY, QueryString.ENTRY, Match.ENTRY); + List entries = new ArrayList<>(); + + entries.add(MatchQueryPredicate.ENTRY); + entries.add(MultiMatchQueryPredicate.ENTRY); + entries.add(QueryString.ENTRY); + entries.add(Match.ENTRY); + + if (EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()) { + entries.add(Kql.ENTRY); + } + + return Collections.unmodifiableList(entries); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java new file mode 100644 index 0000000000000..c03902373c02e --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.querydsl.query.KqlQuery; + +import java.io.IOException; +import java.util.List; + +/** + * Full text function that performs a {@link KqlQuery} . + */ +public class Kql extends FullTextFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Kql", Kql::new); + + @FunctionInfo( + returnType = "boolean", + preview = true, + description = "Performs a KQL query. Returns true if the provided KQL query string matches the row.", + examples = { @Example(file = "kql-function", tag = "kql-with-field") } + ) + public Kql( + Source source, + @Param( + name = "query", + type = { "keyword", "text" }, + description = "Query string in KQL query string format." + ) Expression queryString + ) { + super(source, queryString, List.of(queryString)); + } + + private Kql(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(query()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Kql(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Kql::new, query()); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 9f574ee8005b2..3d6c35e914294 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -30,8 +30,8 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Queries; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; -import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; @@ -252,10 +252,10 @@ static boolean canPushToSource(Expression exp, LucenePushdownPredicates lucenePu && Expressions.foldable(cidrMatch.matches()); } else if (exp instanceof SpatialRelatesFunction spatial) { return canPushSpatialFunctionToSource(spatial, lucenePushdownPredicates); - } else if (exp instanceof QueryString) { - return true; } else if (exp instanceof Match mf) { return mf.field() instanceof FieldAttribute && DataType.isString(mf.field().dataType()); + } else if (exp instanceof FullTextFunction) { + return true; } return false; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index 6fac7bab2bd80..1580b77931240 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Check; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; @@ -47,6 +48,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.querydsl.query.KqlQuery; import org.elasticsearch.xpack.esql.querydsl.query.SpatialRelatesQuery; import org.elasticsearch.xpack.versionfield.Version; @@ -89,6 +91,7 @@ public final class EsqlExpressionTranslators { new ExpressionTranslators.MultiMatches(), new MatchFunctionTranslator(), new QueryStringFunctionTranslator(), + new KqlFunctionTranslator(), new Scalars() ); @@ -538,4 +541,11 @@ protected Query asQuery(QueryString queryString, TranslatorHandler handler) { return new QueryStringQuery(queryString.source(), queryString.queryAsText(), Map.of(), Map.of()); } } + + public static class KqlFunctionTranslator extends ExpressionTranslator { + @Override + protected Query asQuery(Kql kqlFunction, TranslatorHandler handler) { + return new KqlQuery(kqlFunction.source(), kqlFunction.queryAsText()); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java new file mode 100644 index 0000000000000..c388a131b9ab6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.querydsl.query; + +import org.elasticsearch.core.Booleans; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.kql.query.KqlQueryBuilder; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.function.BiConsumer; + +import static java.util.Map.entry; + +public class KqlQuery extends Query { + + private static final Map> BUILDER_APPLIERS = Map.ofEntries( + entry(KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), KqlQueryBuilder::timeZone), + entry(KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), KqlQueryBuilder::defaultField), + entry(KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), (qb, s) -> qb.caseInsensitive(Booleans.parseBoolean(s))) + ); + + private final String query; + + private final Map options; + + // dedicated constructor for QueryTranslator + public KqlQuery(Source source, String query) { + this(source, query, null); + } + + public KqlQuery(Source source, String query, Map options) { + super(source); + this.query = query; + this.options = options == null ? Collections.emptyMap() : options; + } + + @Override + public QueryBuilder asBuilder() { + final KqlQueryBuilder queryBuilder = new KqlQueryBuilder(query); + options.forEach((k, v) -> { + if (BUILDER_APPLIERS.containsKey(k)) { + BUILDER_APPLIERS.get(k).accept(queryBuilder, v); + } else { + throw new IllegalArgumentException("illegal kql query option [" + k + "]"); + } + }); + return queryBuilder; + } + + public String query() { + return query; + } + + public Map options() { + return options; + } + + @Override + public int hashCode() { + return Objects.hash(query, options); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + + KqlQuery other = (KqlQuery) obj; + return Objects.equals(query, other.query) && Objects.equals(options, other.options); + } + + @Override + protected String innerToString() { + return query; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 012720db9efd9..010a60ef7da15 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -257,6 +257,10 @@ public final void test() throws Throwable { "can't use MATCH function in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_FUNCTION.capabilityName()) ); + assumeFalse( + "can't use KQL function in csv tests", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.KQL_FUNCTION.capabilityName()) + ); assumeFalse( "lookup join disabled for csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP.capabilityName()) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 7b2f85b80b3b6..f25b19c4e5d1c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1263,11 +1263,74 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { ); } + public void testKqlFunctionsNotAllowedAfterCommands() throws Exception { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + // Source commands + assertEquals("1:13: [KQL] function cannot be used after SHOW", error("show info | where kql(\"8.16.0\")")); + assertEquals("1:17: [KQL] function cannot be used after ROW", error("row a= \"Anna\" | where kql(\"Anna\")")); + + // Processing commands + assertEquals( + "1:43: [KQL] function cannot be used after DISSECT", + error("from test | dissect first_name \"%{foo}\" | where kql(\"Connection\")") + ); + assertEquals("1:27: [KQL] function cannot be used after DROP", error("from test | drop emp_no | where kql(\"Anna\")")); + assertEquals( + "1:71: [KQL] function cannot be used after ENRICH", + error("from test | enrich languages on languages with lang = language_name | where kql(\"Anna\")") + ); + assertEquals("1:26: [KQL] function cannot be used after EVAL", error("from test | eval z = 2 | where kql(\"Anna\")")); + assertEquals( + "1:44: [KQL] function cannot be used after GROK", + error("from test | grok last_name \"%{WORD:foo}\" | where kql(\"Anna\")") + ); + assertEquals("1:27: [KQL] function cannot be used after KEEP", error("from test | keep emp_no | where kql(\"Anna\")")); + assertEquals("1:24: [KQL] function cannot be used after LIMIT", error("from test | limit 10 | where kql(\"Anna\")")); + assertEquals("1:35: [KQL] function cannot be used after MV_EXPAND", error("from test | mv_expand last_name | where kql(\"Anna\")")); + assertEquals( + "1:45: [KQL] function cannot be used after RENAME", + error("from test | rename last_name as full_name | where kql(\"Anna\")") + ); + assertEquals( + "1:52: [KQL] function cannot be used after STATS", + error("from test | STATS c = COUNT(emp_no) BY languages | where kql(\"Anna\")") + ); + + // Some combination of processing commands + assertEquals("1:38: [KQL] function cannot be used after LIMIT", error("from test | keep emp_no | limit 10 | where kql(\"Anna\")")); + assertEquals( + "1:46: [KQL] function cannot be used after MV_EXPAND", + error("from test | limit 10 | mv_expand last_name | where kql(\"Anna\")") + ); + assertEquals( + "1:52: [KQL] function cannot be used after KEEP", + error("from test | mv_expand last_name | keep last_name | where kql(\"Anna\")") + ); + assertEquals( + "1:77: [KQL] function cannot be used after RENAME", + error("from test | STATS c = COUNT(emp_no) BY languages | rename c as total_emps | where kql(\"Anna\")") + ); + assertEquals( + "1:54: [KQL] function cannot be used after DROP", + error("from test | rename last_name as name | drop emp_no | where kql(\"Anna\")") + ); + } + public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception { assertEquals("1:9: [QSTR] function is only supported in WHERE commands", error("row a = qstr(\"Anna\")")); checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")", "function"); } + public void testKqlFunctionOnlyAllowedInWhere() throws Exception { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + assertEquals("1:9: [KQL] function is only supported in WHERE commands", error("row a = kql(\"Anna\")")); + checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function"); + } + public void testMatchFunctionOnlyAllowedInWhere() throws Exception { checkFullTextFunctionsOnlyAllowedInWhere("MATCH", "match(first_name, \"Anna\")", "function"); } @@ -1309,10 +1372,29 @@ public void testQueryStringFunctionArgNotNullOrConstant() throws Exception { // Other value types are tested in QueryStringFunctionTests } + public void testKqlFunctionArgNotNullOrConstant() throws Exception { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + assertEquals( + "1:19: argument of [kql(first_name)] must be a constant, received [first_name]", + error("from test | where kql(first_name)") + ); + assertEquals("1:19: argument of [kql(null)] cannot be null, received [null]", error("from test | where kql(null)")); + // Other value types are tested in KqlFunctionTests + } + public void testQueryStringWithDisjunctions() { checkWithDisjunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); } + public void testKqlFunctionWithDisjunctions() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function"); + } + public void testMatchFunctionWithDisjunctions() { checkWithDisjunctions("MATCH", "match(first_name, \"Anna\")", "function"); } @@ -1368,6 +1450,13 @@ public void testQueryStringFunctionWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); } + public void testKqlFunctionWithNonBooleanFunctions() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function"); + } + public void testMatchFunctionWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions("MATCH", "match(first_name, \"Anna\")", "function"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java new file mode 100644 index 0000000000000..d97be6b169eef --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.junit.BeforeClass; + +import java.util.List; +import java.util.function.Supplier; + +public class KqlTests extends NoneFieldFullTextFunctionTestCase { + @BeforeClass + protected static void ensureKqlFunctionEnabled() { + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + } + + public KqlTests(@Name("TestCase") Supplier testCaseSupplier) { + super(testCaseSupplier); + } + + @ParametersFactory + public static Iterable parameters() { + return generateParameters(); + } + + @Override + protected Expression build(Source source, List args) { + return new Kql(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/NoneFieldFullTextFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/NoneFieldFullTextFunctionTestCase.java new file mode 100644 index 0000000000000..383cb8671053d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/NoneFieldFullTextFunctionTestCase.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.LinkedList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +public abstract class NoneFieldFullTextFunctionTestCase extends AbstractFunctionTestCase { + + public NoneFieldFullTextFunctionTestCase(Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + public final void testFold() { + Expression expression = buildLiteralExpression(testCase); + if (testCase.getExpectedTypeError() != null) { + assertTypeResolutionFailure(expression); + return; + } + assertFalse("expected resolved", expression.typeResolved().unresolved()); + } + + protected static Iterable generateParameters() { + List suppliers = new LinkedList<>(); + for (DataType strType : DataType.stringTypes()) { + suppliers.add( + new TestCaseSupplier( + "<" + strType + ">", + List.of(strType), + () -> testCase(strType, randomAlphaOfLengthBetween(1, 10), equalTo(true)) + ) + ); + } + List errorsSuppliers = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); + // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests + return parameterSuppliersFromTypedData(errorsSuppliers.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList()); + } + + private static TestCaseSupplier.TestCase testCase(DataType strType, String str, Matcher matcher) { + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(new BytesRef(str), strType, "query")), + "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", + DataType.BOOLEAN, + matcher + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java index b4b4ebcaacde6..f573e59ab205a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java @@ -10,61 +10,24 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.hamcrest.Matcher; -import java.util.LinkedList; import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; - @FunctionName("qstr") -public class QueryStringTests extends AbstractFunctionTestCase { +public class QueryStringTests extends NoneFieldFullTextFunctionTestCase { public QueryStringTests(@Name("TestCase") Supplier testCaseSupplier) { - this.testCase = testCaseSupplier.get(); + super(testCaseSupplier); } @ParametersFactory public static Iterable parameters() { - List suppliers = new LinkedList<>(); - for (DataType strType : DataType.stringTypes()) { - suppliers.add( - new TestCaseSupplier( - "<" + strType + ">", - List.of(strType), - () -> testCase(strType, randomAlphaOfLengthBetween(1, 10), equalTo(true)) - ) - ); - } - List errorsSuppliers = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); - // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests - return parameterSuppliersFromTypedData(errorsSuppliers.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList()); - } - - public final void testFold() { - Expression expression = buildLiteralExpression(testCase); - if (testCase.getExpectedTypeError() != null) { - assertTypeResolutionFailure(expression); - return; - } - assertFalse("expected resolved", expression.typeResolved().unresolved()); - } - - private static TestCaseSupplier.TestCase testCase(DataType strType, String str, Matcher matcher) { - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(new BytesRef(str), strType, "query")), - "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", - DataType.BOOLEAN, - matcher - ); + return generateParameters(); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 269b4806680a6..4612ccb425ba2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.EsqlTestUtils.TestSearchStats; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; @@ -62,6 +63,7 @@ import org.elasticsearch.xpack.esql.stats.Metrics; import org.elasticsearch.xpack.esql.stats.SearchContextStats; import org.elasticsearch.xpack.esql.stats.SearchStats; +import org.elasticsearch.xpack.kql.query.KqlQueryBuilder; import org.junit.Before; import java.io.IOException; @@ -678,7 +680,7 @@ public void testMatchFunctionMultipleWhereClauses() { * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}}, * {"match":{"first_name":{"query":"John"}}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324] */ - public void testMatchFunctionMultipleQstrClauses() { + public void testMatchFunctionMultipleMatchClauses() { String queryText = """ from test | where match(last_name, "Smith") and match(first_name, "John") @@ -698,6 +700,182 @@ public void testMatchFunctionMultipleQstrClauses() { assertThat(query.query().toString(), is(expected.toString())); } + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"kql":{"query":"last_name: Smith"}}] + */ + public void testKqlFunction() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + var plan = plannerOptimizer.plan(""" + from test + | where kql("last_name: Smith") + """, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + var expected = kqlQueryBuilder("last_name: Smith"); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418],false] + * \_ProjectExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418]] + * \_FieldExtractExec[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"kql":{"query":"last_name: Smith"}} + * ,{"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010"}}], + * "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324] + */ + public void testKqlFunctionConjunctionWhereOperands() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + String queryText = """ + from test + | where kql("last_name: Smith") and emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 36, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var kqlQuery = kqlQueryBuilder("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(kqlQuery).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], + * false] + * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] + * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"kql":{"query":"last_name: Smith"}},{ + * "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, + * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354] + */ + public void testKqlFunctionWithFunctionsPushedToLucene() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + String queryText = """ + from test + | where kql("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") + """; + var analyzer = makeAnalyzer("mapping-all-types.json"); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 36, "cidr_match(ip, \"127.0.0.1/32\")"); + var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var kqlQuery = kqlQueryBuilder("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(kqlQuery).must(terms); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162],false] + * \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]] + * \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], + * query[{"bool":{"must":[{"kql":{"query":"last_name: Smith"}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}], + * "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324] + */ + public void testKqlFunctionMultipleWhereClauses() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + String queryText = """ + from test + | where kql("last_name: Smith") + | where emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(3, 8, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var kqlQuery = kqlQueryBuilder("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(kqlQuery).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"bool": {"must":[ + * {"kql":{"query":"last_name: Smith"}}, + * {"kql":{"query":"emp_no > 10010"}}],"boost":1.0}}] + */ + public void testKqlFunctionMultipleKqlClauses() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + String queryText = """ + from test + | where kql("last_name: Smith") and kql("emp_no > 10010") + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + var kqlQueryLeft = kqlQueryBuilder("last_name: Smith"); + var kqlQueryRight = kqlQueryBuilder("emp_no > 10010"); + var expected = QueryBuilders.boolQuery().must(kqlQueryLeft).must(kqlQueryRight); + assertThat(query.query().toString(), is(expected.toString())); + } + // optimizer doesn't know yet how to break down different multi count public void testCountFieldsAndAllWithFilter() { var plan = plannerOptimizer.plan(""" @@ -1166,4 +1344,8 @@ private Stat queryStatsFor(PhysicalPlan plan) { protected List filteredWarnings() { return withDefaultLimitWarning(super.filteredWarnings()); } + + private static KqlQueryBuilder kqlQueryBuilder(String query) { + return new KqlQueryBuilder(query); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java new file mode 100644 index 0000000000000..8dfb50f84ac1e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.querydsl.query; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.tree.SourceTests; +import org.elasticsearch.xpack.esql.core.util.StringUtils; +import org.elasticsearch.xpack.kql.query.KqlQueryBuilder; + +import java.time.ZoneId; +import java.time.zone.ZoneRulesException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.hamcrest.Matchers.equalTo; + +public class KqlQueryTests extends ESTestCase { + static KqlQuery randomKqkQueryQuery() { + Map options = new HashMap<>(); + + if (randomBoolean()) { + options.put(KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), String.valueOf(randomBoolean())); + } + + if (randomBoolean()) { + options.put(KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), randomIdentifier()); + } + + if (randomBoolean()) { + options.put(KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), randomZone().getId()); + } + + return new KqlQuery(SourceTests.randomSource(), randomAlphaOfLength(5), Collections.unmodifiableMap(options)); + } + + public void testEqualsAndHashCode() { + for (int runs = 0; runs < 100; runs++) { + checkEqualsAndHashCode(randomKqkQueryQuery(), KqlQueryTests::copy, KqlQueryTests::mutate); + } + } + + private static KqlQuery copy(KqlQuery query) { + return new KqlQuery(query.source(), query.query(), query.options()); + } + + private static KqlQuery mutate(KqlQuery query) { + List> options = Arrays.asList( + q -> new KqlQuery(SourceTests.mutate(q.source()), q.query(), q.options()), + q -> new KqlQuery(q.source(), randomValueOtherThan(q.query(), () -> randomAlphaOfLength(5)), q.options()), + q -> new KqlQuery(q.source(), q.query(), mutateOptions(q.options())) + ); + + return randomFrom(options).apply(query); + } + + private static Map mutateOptions(Map options) { + Map mutatedOptions = new HashMap<>(options); + if (options.isEmpty() == false && randomBoolean()) { + mutatedOptions = options.entrySet() + .stream() + .filter(entry -> randomBoolean()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + while (mutatedOptions.equals(options)) { + if (randomBoolean()) { + mutatedOptions = mutateOption( + mutatedOptions, + KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), + () -> String.valueOf(randomBoolean()) + ); + } + + if (randomBoolean()) { + mutatedOptions = mutateOption( + mutatedOptions, + KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), + () -> randomIdentifier() + ); + } + + if (randomBoolean()) { + mutatedOptions = mutateOption( + mutatedOptions, + KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), + () -> randomZone().getId() + ); + } + } + + return Collections.unmodifiableMap(mutatedOptions); + } + + private static Map mutateOption(Map options, String optionName, Supplier valueSupplier) { + options = new HashMap<>(options); + options.put(optionName, randomValueOtherThan(options.get(optionName), valueSupplier)); + return options; + } + + public void testQueryBuilding() { + KqlQueryBuilder qb = getBuilder(Map.of("case_insensitive", "false")); + assertThat(qb.caseInsensitive(), equalTo(false)); + + qb = getBuilder(Map.of("case_insensitive", "false", "time_zone", "UTC", "default_field", "foo")); + assertThat(qb.caseInsensitive(), equalTo(false)); + assertThat(qb.timeZone(), equalTo(ZoneId.of("UTC"))); + assertThat(qb.defaultField(), equalTo("foo")); + + Exception e = expectThrows(IllegalArgumentException.class, () -> getBuilder(Map.of("pizza", "yummy"))); + assertThat(e.getMessage(), equalTo("illegal kql query option [pizza]")); + + e = expectThrows(ZoneRulesException.class, () -> getBuilder(Map.of("time_zone", "aoeu"))); + assertThat(e.getMessage(), equalTo("Unknown time-zone ID: aoeu")); + } + + private static KqlQueryBuilder getBuilder(Map options) { + final Source source = new Source(1, 1, StringUtils.EMPTY); + final KqlQuery kqlQuery = new KqlQuery(source, "eggplant", options); + return (KqlQueryBuilder) kqlQuery.asBuilder(); + } + + public void testToString() { + final Source source = new Source(1, 1, StringUtils.EMPTY); + final KqlQuery kqlQuery = new KqlQuery(source, "eggplant", Map.of()); + assertEquals("KqlQuery@1:2[eggplant]", kqlQuery.toString()); + } +} diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java index 5dff9126b6be4..e2817665d8f79 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -26,6 +27,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.kql.parser.KqlParser; import org.elasticsearch.xpack.kql.parser.KqlParsingContext; +import org.elasticsearch.xpack.kql.parser.KqlParsingException; import java.io.IOException; import java.time.ZoneId; @@ -37,9 +39,9 @@ public class KqlQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "kql"; public static final ParseField QUERY_FIELD = new ParseField("query"); - private static final ParseField CASE_INSENSITIVE_FIELD = new ParseField("case_insensitive"); - private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); - private static final ParseField DEFAULT_FIELD_FIELD = new ParseField("default_field"); + public static final ParseField CASE_INSENSITIVE_FIELD = new ParseField("case_insensitive"); + public static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); + public static final ParseField DEFAULT_FIELD_FIELD = new ParseField("default_field"); private static final Logger log = LogManager.getLogger(KqlQueryBuilder.class); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, a -> { @@ -151,12 +153,16 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep @Override protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) throws IOException { - KqlParser parser = new KqlParser(); - QueryBuilder rewrittenQuery = parser.parseKqlQuery(query, createKqlParserContext(context)); + try { + KqlParser parser = new KqlParser(); + QueryBuilder rewrittenQuery = parser.parseKqlQuery(query, createKqlParserContext(context)); - log.trace(() -> Strings.format("KQL query %s translated to Query DSL: %s", query, Strings.toString(rewrittenQuery))); + log.trace(() -> Strings.format("KQL query %s translated to Query DSL: %s", query, Strings.toString(rewrittenQuery))); - return rewrittenQuery; + return rewrittenQuery; + } catch (KqlParsingException e) { + throw new QueryShardException(context, "Failed to parse KQL query [{}]", e, query); + } } @Override diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 72c7c51655378..f7dd979540afa 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -92,7 +92,7 @@ setup: - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation. - - length: {esql.functions: 121} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 122} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version":