[ES|QL] Enable KQL function as a tech preview

docs/changelog/119730.yaml

@@ -0,0 +1,5 @@
+pr: 119730
+summary: Enable KQL function as a tech preview
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/esql/esql-limitations.asciidoc

@@ -150,7 +150,7 @@ FROM books
 
 Note that, because of <<esql-limitations-text-fields,the way {esql} treats `text` values>>,
 any queries on `text` fields that do not explicitly use the full-text functions,
-<<esql-match>> or <<esql-qstr>>, will behave as if the fields are actually `keyword` fields:
+<<esql-match>>, <<esql-qstr>> or <<esql-kql>>, will behave as if the fields are actually `keyword` fields:
 they are case-sensitive and need to match the full string.
 
 [discrete]

docs/reference/esql/functions/search-functions.asciidoc

@@ -18,9 +18,11 @@ See <<esql-limitations-full-text-search,full text search limitations>> for infor
 {esql} supports these full-text search functions:
 
 // tag::search_list[]
+* experimental:[] <<esql-kql>>
 * experimental:[] <<esql-match>>
 * experimental:[] <<esql-qstr>>
 // end::search_list[]
 
+include::layout/kql.asciidoc[]
 include::layout/match.asciidoc[]
 include::layout/qstr.asciidoc[]

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -155,6 +155,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC = def(8_820_00_0);
     public static final TransportVersion REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX = def(8_821_00_0);
     public static final TransportVersion ELASTIC_INFERENCE_SERVICE_UNIFIED_CHAT_COMPLETIONS_INTEGRATION = def(8_822_00_0);
+    public static final TransportVersion KQL_QUERY_TECH_PREVIEW = def(8_823_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java

@@ -9,8 +9,6 @@
 
 package org.elasticsearch.rest.action.search;
 
-import org.elasticsearch.Build;
-
 import java.util.HashSet;
 import java.util.Set;
 
@@ -60,9 +58,7 @@ private SearchCapabilities() {}
         capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE);
         capabilities.add(MOVING_FN_RIGHT_MATH);
         capabilities.add(K_DEFAULT_TO_SIZE);
-        if (Build.current().isSnapshot()) {
-            capabilities.add(KQL_QUERY_SUPPORTED);
-        }
+        capabilities.add(KQL_QUERY_SUPPORTED);
         capabilities.add(HIGHLIGHT_MAX_ANALYZED_OFFSET_DEFAULT);
         CAPABILITIES = Set.copyOf(capabilities);
     }

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java

@@ -15,10 +15,8 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.kql.KqlPlugin;
 import org.junit.Before;
-import org.junit.BeforeClass;
 
 import java.util.Collection;
 import java.util.List;
@@ -27,12 +25,6 @@
 import static org.hamcrest.CoreMatchers.containsString;
 
 public class KqlFunctionIT extends AbstractEsqlIntegTestCase {
-
-    @BeforeClass
-    protected static void ensureKqlFunctionEnabled() {
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-    }
-
     @Before
     public void setupIndex() {
         createAndPopulateIndex();

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -452,7 +452,7 @@ public enum Cap {
         /**
          * KQL function
          */
-        KQL_FUNCTION(Build.current().isSnapshot()),
+        KQL_FUNCTION,
 
         /**
          * Hash function

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java

@@ -424,7 +424,10 @@ private static FunctionDefinition[][] functions() {
                 def(MvSum.class, MvSum::new, "mv_sum"),
                 def(Split.class, Split::new, "split") },
             // fulltext functions
-            new FunctionDefinition[] { def(Match.class, bi(Match::new), "match"), def(QueryString.class, uni(QueryString::new), "qstr") } };
+            new FunctionDefinition[] {
+                def(Kql.class, uni(Kql::new), "kql"),
+                def(Match.class, bi(Match::new), "match"),
+                def(QueryString.class, uni(QueryString::new), "qstr") } };
 
     }
 
@@ -434,7 +437,6 @@ private static FunctionDefinition[][] snapshotFunctions() {
                 // The delay() function is for debug/snapshot environments only and should never be enabled in a non-snapshot build.
                 // This is an experimental function and can be removed without notice.
                 def(Delay.class, Delay::new, "delay"),
-                def(Kql.class, uni(Kql::new), "kql"),
                 def(Rate.class, Rate::withUnresolvedTimestamp, "rate"),
                 def(Term.class, bi(Term::new), "term") } };
     }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java

@@ -25,10 +25,8 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(MultiMatchQueryPredicate.ENTRY);
         entries.add(QueryString.ENTRY);
         entries.add(Match.ENTRY);
+        entries.add(Kql.ENTRY);
 
-        if (EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()) {
-            entries.add(Kql.ENTRY);
-        }
         if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) {
             entries.add(Term.ENTRY);
         }

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java

@@ -1288,9 +1288,6 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception {
     }
 
     public void testKqlFunctionsNotAllowedAfterCommands() throws Exception {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         // Source commands
         assertEquals("1:13: [KQL] function cannot be used after SHOW", error("show info | where kql(\"8.16.0\")"));
         assertEquals("1:17: [KQL] function cannot be used after ROW", error("row a= \"Anna\" | where kql(\"Anna\")"));
@@ -1348,9 +1345,6 @@ public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception {
     }
 
     public void testKqlFunctionOnlyAllowedInWhere() throws Exception {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         assertEquals("1:9: [KQL] function is only supported in WHERE commands", error("row a = kql(\"Anna\")"));
         checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function");
     }
@@ -1402,9 +1396,6 @@ public void testQueryStringFunctionArgNotNullOrConstant() throws Exception {
     }
 
     public void testKqlFunctionArgNotNullOrConstant() throws Exception {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         assertEquals(
             "1:19: argument of [kql(first_name)] must be a constant, received [first_name]",
             error("from test | where kql(first_name)")
@@ -1418,9 +1409,6 @@ public void testQueryStringWithDisjunctions() {
     }
 
     public void testKqlFunctionWithDisjunctions() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function");
     }
 
@@ -1463,8 +1451,6 @@ public void testFullTextFunctionsDisjunctions() {
         checkWithFullTextFunctionsDisjunctions("MATCH", "match(last_name, \"Smith\")", "function");
         checkWithFullTextFunctionsDisjunctions(":", "last_name : \"Smith\"", "operator");
         checkWithFullTextFunctionsDisjunctions("QSTR", "qstr(\"last_name: Smith\")", "function");
-
-        assumeTrue("KQL function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
         checkWithFullTextFunctionsDisjunctions("KQL", "kql(\"last_name: Smith\")", "function");
     }
 
@@ -1493,9 +1479,6 @@ public void testQueryStringFunctionWithNonBooleanFunctions() {
     }
 
     public void testKqlFunctionWithNonBooleanFunctions() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function");
     }
 

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java

@@ -10,21 +10,14 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.junit.BeforeClass;
 
 import java.util.List;
 import java.util.function.Supplier;
 
 public class KqlTests extends NoneFieldFullTextFunctionTestCase {
-    @BeforeClass
-    protected static void ensureKqlFunctionEnabled() {
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-    }
-
     public KqlTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
         super(testCaseSupplier);
     }

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java

@@ -796,9 +796,6 @@ public void testMatchFunctionMultipleMatchClauses() {
      *       \_EsQueryExec[test], indexMode[standard], query[{"kql":{"query":"last_name: Smith"}}]
      */
     public void testKqlFunction() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         var plan = plannerOptimizer.plan("""
             from test
             | where kql("last_name: Smith")
@@ -827,9 +824,6 @@ public void testKqlFunction() {
      *        "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324]
      */
     public void testKqlFunctionConjunctionWhereOperands() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith") and emp_no > 10010
@@ -864,9 +858,6 @@ public void testKqlFunctionConjunctionWhereOperands() {
      *       "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354]
      */
     public void testKqlFunctionWithFunctionsPushedToLucene() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith") and cidr_match(ip, "127.0.0.1/32")
@@ -902,9 +893,6 @@ public void testKqlFunctionWithFunctionsPushedToLucene() {
      *       "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324]
      */
     public void testKqlFunctionMultipleWhereClauses() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith")
@@ -939,9 +927,6 @@ public void testKqlFunctionMultipleWhereClauses() {
      *       {"kql":{"query":"emp_no > 10010"}}],"boost":1.0}}]
      */
     public void testKqlFunctionMultipleKqlClauses() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith") and kql("emp_no > 10010")

x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java

@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.kql;
 
-import org.elasticsearch.Build;
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
@@ -18,10 +17,6 @@
 public class KqlPlugin extends Plugin implements SearchPlugin, ExtensiblePlugin {
     @Override
     public List<QuerySpec<?>> getQueries() {
-        if (Build.current().isSnapshot()) {
-            return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent));
-        }
-
-        return List.of();
+        return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent));
     }
 }

x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java

@@ -97,7 +97,7 @@ public static KqlQueryBuilder fromXContent(XContentParser parser) {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.KQL_QUERY_ADDED;
+        return TransportVersions.KQL_QUERY_TECH_PREVIEW;
     }
 
     public String queryString() {

x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java

@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.kql.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Build;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -22,7 +21,6 @@
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.xpack.kql.KqlPlugin;
 import org.hamcrest.Matchers;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -36,12 +34,6 @@
 import static org.hamcrest.Matchers.nullValue;
 
 public class KqlQueryBuilderTests extends AbstractQueryTestCase<KqlQueryBuilder> {
-    @BeforeClass
-    protected static void ensureSnapshotBuild() {
-        assumeTrue("requires snapshot builds", Build.current().isSnapshot());
-    }
-
-    @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return List.of(KqlPlugin.class);
     }

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml

@@ -163,4 +163,4 @@ setup:
   - match: {esql.functions.cos: $functions_cos}
   - gt: {esql.functions.to_long: $functions_to_long}
   - match: {esql.functions.coalesce: $functions_coalesce}
-  - length: {esql.functions: 129} # check the "sister" test above for a likely update to the same esql.functions length check
+  - length: {esql.functions: 130} # check the "sister" test above for a likely update to the same esql.functions length check