diff --git a/docs/reference/esql/functions/description/qstr.asciidoc b/docs/reference/esql/functions/description/qstr.asciidoc new file mode 100644 index 0000000000000..5ce9316405ad2 --- /dev/null +++ b/docs/reference/esql/functions/description/qstr.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a query string query. Returns true if the provided query string matches the row. diff --git a/docs/reference/esql/functions/examples/qstr.asciidoc b/docs/reference/esql/functions/examples/qstr.asciidoc new file mode 100644 index 0000000000000..003373c84c029 --- /dev/null +++ b/docs/reference/esql/functions/examples/qstr.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/qstr-function.csv-spec[tag=qstr-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/qstr-function.csv-spec[tag=qstr-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json new file mode 100644 index 0000000000000..dfa3dfd3818ad --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/qstr.json @@ -0,0 +1,36 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "qstr", + "description" : "Performs a query string query. Returns true if the provided query string matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Query string in Lucene query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Query string in Lucene query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "from books \n| where qstr(\"author: Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;" + ], + "preview" : true +} diff --git a/docs/reference/esql/functions/kibana/docs/qstr.md b/docs/reference/esql/functions/kibana/docs/qstr.md new file mode 100644 index 0000000000000..37b5777623185 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/qstr.md @@ -0,0 +1,14 @@ + + +### QSTR +Performs a query string query. Returns true if the provided query string matches the row. + +``` +from books +| where qstr("author: Faulkner") +| keep book_no, author +| sort book_no +| limit 5; +``` diff --git a/docs/reference/esql/functions/layout/qstr.asciidoc b/docs/reference/esql/functions/layout/qstr.asciidoc new file mode 100644 index 0000000000000..715a11089f0d4 --- /dev/null +++ b/docs/reference/esql/functions/layout/qstr.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-qstr]] +=== `QSTR` + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/qstr.svg[Embedded,opts=inline] + +include::../parameters/qstr.asciidoc[] +include::../description/qstr.asciidoc[] +include::../types/qstr.asciidoc[] +include::../examples/qstr.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/qstr.asciidoc b/docs/reference/esql/functions/parameters/qstr.asciidoc new file mode 100644 index 0000000000000..e51096084f2f3 --- /dev/null +++ b/docs/reference/esql/functions/parameters/qstr.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`query`:: +Query string in Lucene query string format. diff --git a/docs/reference/esql/functions/signature/qstr.svg b/docs/reference/esql/functions/signature/qstr.svg new file mode 100644 index 0000000000000..0d3841b071cef --- /dev/null +++ b/docs/reference/esql/functions/signature/qstr.svg @@ -0,0 +1 @@ +QSTR(query) diff --git a/docs/reference/esql/functions/types/qstr.asciidoc b/docs/reference/esql/functions/types/qstr.asciidoc new file mode 100644 index 0000000000000..866a39e925665 --- /dev/null +++ b/docs/reference/esql/functions/types/qstr.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +query | result +keyword | boolean +text | boolean +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec new file mode 100644 index 0000000000000..2f6313925032e --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec @@ -0,0 +1,116 @@ +############################################### +# Tests for QSTR function +# + +qstrWithField +required_capability: qstr_function + +// tag::qstr-with-field[] +from books +| where qstr("author: Faulkner") +| keep book_no, author +| sort book_no +| limit 5; +// end::qstr-with-field[] + +// tag::qstr-with-field-result[] +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; +// end::qstr-with-field-result[] + +qstrWithMultipleFields +required_capability: qstr_function + +from books +| where qstr("title:Return* AND author:*Tolkien") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +qstrWithQueryExpressions +required_capability: qstr_function + +from books +| where qstr(CONCAT("title:Return*", " AND author:*Tolkien")) +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +qstrWithDisjunction +required_capability: qstr_function + +from books +| where qstr("title:Return") or year > 2020 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +6818 | Hadji Murad +7350 | Return of the Shadow +; + +qstrWithConjunction +required_capability: qstr_function + +from books +| where qstr("title: Rings") and ratings > 4.6 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + +qstrWithFunctionPushedToLucene +required_capability: qstr_function + +from hosts +| where qstr("host: beta") and cidr_match(ip1, "127.0.0.2/32", "127.0.0.3/32") +| keep card, host, ip0, ip1; +ignoreOrder:true + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth1 |beta |127.0.0.1 |127.0.0.2 +; + +qstrWithFunctionNotPushedToLucene +required_capability: qstr_function + +from books +| where qstr("title: rings") and length(description) > 600 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2675 | The Lord of the Rings - Boxed Set +2714 | Return of the King Being the Third Part of The Lord of the Rings +; + +qstrWithMultipleWhereClauses +required_capability: qstr_function + +from books +| where qstr("title: rings") +| where qstr("year: [1 TO 2005]") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java new file mode 100644 index 0000000000000..e6f11ca1f44d2 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plugin; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; +import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; +import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.junit.Before; + +import java.util.List; + +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class QueryStringFunctionIT extends AbstractEsqlIntegTestCase { + + @Before + public void setupIndex() { + createAndPopulateIndex(); + } + + @Override + protected EsqlQueryResponse run(EsqlQueryRequest request) { + assumeTrue("qstr function available in snapshot builds only", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + return super.run(request); + } + + public void testSimpleQueryString() { + var query = """ + FROM test + | WHERE qstr("content: dog") + | KEEP id + | SORT id + """; + + try (var resp = run(query)) { + assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id"))); + assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER"))); + // values + List> values = getValuesList(resp); + assertMap(values, matchesList().item(List.of(1)).item(List.of(3)).item(List.of(4)).item(List.of(5))); + } + } + + public void testMultiFieldQueryString() { + var query = """ + FROM test + | WHERE qstr("dog OR canine") + | KEEP id + """; + + try (var resp = run(query)) { + assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id"))); + assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER"))); + // values + List> values = getValuesList(resp); + assertThat(values.size(), equalTo(5)); + } + } + + public void testQueryStringWithinEval() { + var query = """ + FROM test + | EVAL matches_query = qstr("title: fox") + """; + + var error = expectThrows(VerificationException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("[QSTR] function is only supported in WHERE commands")); + } + + public void testInvalidQueryStringEof() { + var query = """ + FROM test + | WHERE qstr("content: ((((dog") + """; + + var error = expectThrows(QueryShardException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("Failed to parse query [content: ((((dog]")); + assertThat(error.getRootCause().getMessage(), containsString("Encountered \"\" at line 1, column 16")); + } + + public void testInvalidQueryStringLexicalError() { + var query = """ + FROM test + | WHERE qstr("/") + """; + + var error = expectThrows(QueryShardException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("Failed to parse query [/]")); + assertThat( + error.getRootCause().getMessage(), + containsString("Lexical error at line 1, column 2. Encountered: (in lexical state 2)") + ); + } + + private void createAndPopulateIndex() { + var indexName = "test"; + var client = client().admin().indices(); + var CreateRequest = client.prepareCreate(indexName) + .setSettings(Settings.builder().put("index.number_of_shards", 1)) + .setMapping("id", "type=integer", "content", "type=text"); + assertAcked(CreateRequest); + client().prepareBulk() + .add( + new IndexRequest(indexName).id("1") + .source("id", 1, "content", "The quick brown animal swiftly jumps over a lazy dog", "title", "A Swift Fox's Journey") + ) + .add( + new IndexRequest(indexName).id("2") + .source("id", 2, "content", "A speedy brown fox hops effortlessly over a sluggish canine", "title", "The Fox's Leap") + ) + .add( + new IndexRequest(indexName).id("3") + .source("id", 3, "content", "Quick and nimble, the fox vaults over the lazy dog", "title", "Brown Fox in Action") + ) + .add( + new IndexRequest(indexName).id("4") + .source( + "id", + 4, + "content", + "A fox that is quick and brown jumps over a dog that is quite lazy", + "title", + "Speedy Animals" + ) + ) + .add( + new IndexRequest(indexName).id("5") + .source( + "id", + 5, + "content", + "With agility, a quick brown fox bounds over a slow-moving dog", + "title", + "Foxes and Canines" + ) + ) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + ensureYellow(indexName); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index b13dd3c095f19..597c349273eb2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -311,6 +311,11 @@ public enum Cap { */ CATEGORIZE(true), + /** + * QSTR function + */ + QSTR_FUNCTION(true), + /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. * https://github.com/elastic/elasticsearch/issues/112704 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 9714d3fce6d9f..c466f9ebb5e53 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -187,6 +188,7 @@ else if (p instanceof Lookup lookup) { checkFilterMatchConditions(p, failures); checkMatchCommand(p, failures); + checkFullTextQueryFunctions(p, failures); }); checkRemoteEnrich(plan, failures); @@ -657,4 +659,31 @@ private static void checkMatchCommand(LogicalPlan plan, Set failures) { } } } + + private static void checkFullTextQueryFunctions(LogicalPlan plan, Set failures) { + if (plan instanceof Filter f) { + Expression condition = f.condition(); + if (condition instanceof FullTextFunction ftf) { + // Similar to cases present in org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters - + // we can't check if it can be pushed down as we don't have yet information about the fields present in the + // StringQueryPredicate + plan.forEachDown(LogicalPlan.class, lp -> { + if ((lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation) == false) { + failures.add( + fail( + plan, + "[{}] function cannot be used after {}", + ftf.functionName(), + lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT) + ) + ); + } + }); + } + } else { + plan.forEachExpression(FullTextFunction.class, ftf -> { + failures.add(fail(ftf, "[{}] function is only supported in WHERE commands", ftf.functionName())); + }); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index ce8c20e4fbf11..5a6430e0fdfad 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Top; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.aggregate.WeightedAvg; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryStringFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; @@ -386,8 +387,10 @@ private FunctionDefinition[][] functions() { private static FunctionDefinition[][] snapshotFunctions() { return new FunctionDefinition[][] { new FunctionDefinition[] { + def(Rate.class, Rate::withUnresolvedTimestamp, "rate"), def(Categorize.class, Categorize::new, "categorize"), - def(Rate.class, Rate::withUnresolvedTimestamp, "rate") } }; + // Full text functions + def(QueryStringFunction.class, QueryStringFunction::new, "qstr") } }; } public EsqlFunctionRegistry snapshotRegistry() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java new file mode 100644 index 0000000000000..54730eec4f317 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.function.Function; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +/** + * Base class for full-text functions that use ES queries to match documents. + * These functions needs to be pushed down to Lucene queries to be executed - there's no Evaluator for them, but depend on + * {@link org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer} to rewrite them into Lucene queries. + */ +public abstract class FullTextFunction extends Function { + public static List getNamedWriteables() { + List entries = new ArrayList<>(); + if (EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()) { + entries.add(QueryStringFunction.ENTRY); + } + return entries; + } + + private final Expression query; + + protected FullTextFunction(Source source, Expression query) { + super(source, singletonList(query)); + this.query = query; + } + + protected FullTextFunction(StreamInput in) throws IOException { + this(Source.readFrom((StreamInput & PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + } + + @Override + public DataType dataType() { + return DataType.BOOLEAN; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + return isString(query(), sourceText(), DEFAULT).and(isNotNullAndFoldable(query(), functionName(), DEFAULT)); + } + + public Expression query() { + return query; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(query); + } + + public abstract Query asQuery(); +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java new file mode 100644 index 0000000000000..fa331acd08655 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Full text function that performs a {@link QueryStringQuery} . + */ +public class QueryStringFunction extends FullTextFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "QStr", + QueryStringFunction::new + ); + + @FunctionInfo( + returnType = "boolean", + preview = true, + description = "Performs a query string query. Returns true if the provided query string matches the row.", + examples = { @Example(file = "qstr-function", tag = "qstr-with-field") } + ) + public QueryStringFunction( + Source source, + @Param( + name = "query", + type = { "keyword", "text" }, + description = "Query string in Lucene query string format." + ) Expression queryString + ) { + super(source, queryString); + } + + private QueryStringFunction(StreamInput in) throws IOException { + super(in); + } + + @Override + public String functionName() { + return "QSTR"; + } + + @Override + public Query asQuery() { + Object queryAsObject = query().fold(); + if (queryAsObject instanceof BytesRef queryAsBytesRef) { + return new QueryStringQuery(source(), queryAsBytesRef.utf8ToString(), Map.of(), null); + } else { + throw new IllegalArgumentException("Query in QSTR needs to be resolved to a string"); + } + } + + @Override + public Expression replaceChildren(List newChildren) { + return new QueryStringFunction(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, QueryStringFunction::new, query()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index f991429651c76..0a71bce2575fa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Queries; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -194,6 +195,8 @@ public static boolean canPushToSource(Expression exp, Predicate return mqp.field() instanceof FieldAttribute && DataType.isString(mqp.field().dataType()); } else if (exp instanceof StringQueryPredicate) { return true; + } else if (exp instanceof FullTextFunction) { + return true; } return false; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index b508dc6556456..18aa2628fdc7c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Check; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils; @@ -84,9 +85,17 @@ public final class EsqlExpressionTranslators { new ExpressionTranslators.StringQueries(), new ExpressionTranslators.Matches(), new ExpressionTranslators.MultiMatches(), + new FullTextFunctions(), new Scalars() ); + public static class FullTextFunctions extends ExpressionTranslator { + @Override + protected Query asQuery(FullTextFunction fullTextFunction, TranslatorHandler handler) { + return fullTextFunction.asQuery(); + } + } + public static Query toQuery(Expression e, TranslatorHandler handler) { Query translation = null; for (ExpressionTranslator translator : QUERY_TRANSLATORS) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 315309fbad677..9b4d51af244b8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -65,6 +65,7 @@ import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -201,6 +202,7 @@ public List getNamedWriteables() { entries.addAll(EsqlScalarFunction.getNamedWriteables()); entries.addAll(AggregateFunction.getNamedWriteables()); entries.addAll(LogicalPlan.getNamedWriteables()); + entries.addAll(FullTextFunction.getNamedWriteables()); entries.addAll(PhysicalPlan.getNamedWriteables()); return entries; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index faf9d04532f1a..3e8d1e4e71562 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -251,6 +251,10 @@ public final void test() throws Throwable { "can't use match command in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_COMMAND.capabilityName()) ); + assumeFalse( + "can't use QSTR function in csv tests", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.QSTR_FUNCTION.capabilityName()) + ); if (Build.current().isSnapshot()) { assertThat( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java index fe74883a0c24f..69a68af91f5db 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; @@ -123,6 +124,7 @@ public static NamedWriteableRegistry writableRegistry() { entries.addAll(AggregateFunction.getNamedWriteables()); entries.addAll(Block.getNamedWriteables()); entries.addAll(LogicalPlan.getNamedWriteables()); + entries.addAll(FullTextFunction.getNamedWriteables()); entries.addAll(PhysicalPlan.getNamedWriteables()); return new NamedWriteableRegistry(entries); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index f8a48d0fd7b4c..0b83b76992546 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1107,6 +1107,86 @@ private void assertMatchCommand(String lineAndColumn, String command, String que assertThat(error(query, defaultAnalyzer, exception), containsString(expectedErrorMessage)); } + public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { + assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + + // Source commands + assertEquals("1:13: [QSTR] function cannot be used after SHOW", error("show info | where qstr(\"8.16.0\")")); + assertEquals("1:17: [QSTR] function cannot be used after ROW", error("row a= \"Anna\" | where qstr(\"Anna\")")); + + // Processing commands + assertEquals( + "1:43: [QSTR] function cannot be used after DISSECT", + error("from test | dissect first_name \"%{foo}\" | where qstr(\"Connection\")") + ); + assertEquals("1:27: [QSTR] function cannot be used after DROP", error("from test | drop emp_no | where qstr(\"Anna\")")); + assertEquals( + "1:71: [QSTR] function cannot be used after ENRICH", + error("from test | enrich languages on languages with lang = language_name | where qstr(\"Anna\")") + ); + assertEquals("1:26: [QSTR] function cannot be used after EVAL", error("from test | eval z = 2 | where qstr(\"Anna\")")); + assertEquals( + "1:44: [QSTR] function cannot be used after GROK", + error("from test | grok last_name \"%{WORD:foo}\" | where qstr(\"Anna\")") + ); + assertEquals("1:27: [QSTR] function cannot be used after KEEP", error("from test | keep emp_no | where qstr(\"Anna\")")); + assertEquals("1:24: [QSTR] function cannot be used after LIMIT", error("from test | limit 10 | where qstr(\"Anna\")")); + assertEquals( + "1:35: [QSTR] function cannot be used after MV_EXPAND", + error("from test | mv_expand last_name | where qstr(\"Anna\")") + ); + assertEquals( + "1:45: [QSTR] function cannot be used after RENAME", + error("from test | rename last_name as full_name | where qstr(\"Anna\")") + ); + assertEquals( + "1:52: [QSTR] function cannot be used after STATS", + error("from test | STATS c = COUNT(emp_no) BY languages | where qstr(\"Anna\")") + ); + + // Some combination of processing commands + assertEquals( + "1:38: [QSTR] function cannot be used after LIMIT", + error("from test | keep emp_no | limit 10 | where qstr(\"Anna\")") + ); + assertEquals( + "1:46: [QSTR] function cannot be used after MV_EXPAND", + error("from test | limit 10 | mv_expand last_name | where qstr(\"Anna\")") + ); + assertEquals( + "1:52: [QSTR] function cannot be used after KEEP", + error("from test | mv_expand last_name | keep last_name | where qstr(\"Anna\")") + ); + assertEquals( + "1:77: [QSTR] function cannot be used after RENAME", + error("from test | STATS c = COUNT(emp_no) BY languages | rename c as total_emps | where qstr(\"Anna\")") + ); + assertEquals( + "1:54: [QSTR] function cannot be used after KEEP", + error("from test | rename last_name as name | keep emp_no | where qstr(\"Anna\")") + ); + } + + public void testQueryStringFunctionsOnlyAllowedInWhere() throws Exception { + assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + + assertEquals("1:22: [QSTR] function is only supported in WHERE commands", error("from test | eval y = qstr(\"Anna\")")); + assertEquals("1:18: [QSTR] function is only supported in WHERE commands", error("from test | sort qstr(\"Connection\") asc")); + assertEquals("1:5: [QSTR] function is only supported in WHERE commands", error("row qstr(\"Connection\")")); + assertEquals( + "1:23: [QSTR] function is only supported in WHERE commands", + error("from test | STATS c = qstr(\"foo\") BY languages") + ); + } + + public void testQueryStringFunctionArgNotNullOrConstant() throws Exception { + assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + + assertEquals("1:19: argument of [QSTR] must be a constant, received [first_name]", error("from test | where qstr(first_name)")); + assertEquals("1:19: argument of [QSTR] cannot be null, received [null]", error("from test | where qstr(null)")); + // Other value types are tested in QueryStringFunctionTests + } + public void testCoalesceWithMixedNumericTypes() { assertEquals( "1:22: second argument of [coalesce(languages, height)] must be [integer], found value [height] type [double]", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java index ab20a5ce0cc6b..a2aa447c748e9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.plan.AbstractNodeSerializationTests; @@ -33,6 +34,7 @@ protected final NamedWriteableRegistry getNamedWriteableRegistry() { entries.addAll(Attribute.getNamedWriteables()); entries.addAll(EsqlScalarFunction.getNamedWriteables()); entries.addAll(AggregateFunction.getNamedWriteables()); + entries.addAll(FullTextFunction.getNamedWriteables()); entries.add(UnsupportedAttribute.ENTRY); entries.add(UnsupportedAttribute.NAMED_EXPRESSION_ENTRY); entries.add(UnsupportedAttribute.EXPRESSION_ENTRY); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java new file mode 100644 index 0000000000000..e622ff5ba2579 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +@FunctionName("qstr") +public class QueryStringFunctionTests extends AbstractFunctionTestCase { + + public QueryStringFunctionTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new LinkedList<>(); + for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) { + suppliers.add( + new TestCaseSupplier( + "<" + strType + ">", + List.of(strType), + () -> testCase(strType, randomAlphaOfLengthBetween(1, 10), equalTo(true)) + ) + ); + } + List errorsSuppliers = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); + // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests + return parameterSuppliersFromTypedData(errorsSuppliers.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList()); + } + + public final void testFold() { + Expression expression = buildLiteralExpression(testCase); + if (testCase.getExpectedTypeError() != null) { + assertTypeResolutionFailure(expression); + return; + } + assertFalse("expected resolved", expression.typeResolved().unresolved()); + } + + private static TestCaseSupplier.TestCase testCase(DataType strType, String str, Matcher matcher) { + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(new BytesRef(str), strType, "query")), + "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", + DataType.BOOLEAN, + matcher + ); + } + + @Override + protected Expression build(Source source, List args) { + return new QueryStringFunction(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index eeb720084e635..2ed0093945837 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; +import org.elasticsearch.xpack.esql.plan.physical.FilterExec; import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -373,6 +374,241 @@ public void testMultiCountAllWithFilter() { assertThat(plan.anyMatch(EsQueryExec.class::isInstance), is(true)); } + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"query_string":{"query":"last_name: Smith","fields":[]}}] + */ + public void testQueryStringFunction() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + var plan = plannerOptimizer.plan(""" + from test + | where qstr("last_name: Smith") + """, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + var expected = QueryBuilders.queryStringQuery("last_name: Smith"); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418],false] + * \_ProjectExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418]] + * \_FieldExtractExec[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}} + * ,{"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010"}}], + * "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324] + */ + public void testQueryStringFunctionConjunctionWhereOperands() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") and emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 37, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(queryString).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n + * ame{f}#7, long_noidx{f}#12, salary{f}#8],false] + * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n + * ame{f}#7, long_noidx{f}#12, salary{f}#8]] + * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"should":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@2:37"}}], + * "boost":1.0}}][_doc{f}#13], limit[1000], sort[] estimatedRowSize[324] + */ + public void testQueryStringFunctionDisjunctionWhereClauses() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") or emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 36, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var expected = QueryBuilders.boolQuery().should(queryString).should(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], + * false] + * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] + * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},{ + * "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, + * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354] + */ + public void testQueryStringFunctionWithFunctionsPushedToLucene() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") + """; + var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")"); + var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(queryString).must(terms); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + *LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n + * ame{f}#7, long_noidx{f}#12, salary{f}#8],false] + * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n + * ame{f}#7, long_noidx{f}#12, salary{f}#8]] + * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, gender{f}#5, job{f}#] + * \_LimitExec[1000[INTEGER]] + * \_FilterExec[LENGTH(first_name{f}#4) > 10[INTEGER]] + * \_FieldExtractExec[first_name{f}#4] + * \_EsQueryExec[test], indexMode[standard], + * query[{"query_string":{"query":"last_name: Smith","fields":[]}}][_doc{f}#13], limit[], sort[] estimatedRowSize[324] + */ + public void testQueryStringFunctionWithFunctionNotPushedDown() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") and length(first_name) > 10 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var firstLimit = as(plan, LimitExec.class); + var exchange = as(firstLimit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var secondLimit = as(field.child(), LimitExec.class); + var filter = as(secondLimit.child(), FilterExec.class); + var fieldExtract = as(filter.child(), FieldExtractExec.class); + var query = as(fieldExtract.child(), EsQueryExec.class); + + var expected = QueryBuilders.queryStringQuery("last_name: Smith"); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162],false] + * \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]] + * \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], + * query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}], + * "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324] + */ + public void testQueryStringFunctionMultipleWhereClauses() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") + | where emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(3, 8, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(queryString).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"bool": + * {"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"query_string":{"query":"emp_no: [10010 TO *]","fields":[]}}],"boost":1.0}}] + */ + public void testQueryStringFunctionMultipleQstrClauses() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") and qstr("emp_no: [10010 TO *]") + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith"); + var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]"); + var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight); + assertThat(query.query().toString(), is(expected.toString())); + } + /** * Expecting * LimitExec[1000[INTEGER]]