[8.x] [ES|QL] Enable KQL function as a tech preview (#119730) (#119954)

afoucret · web-flow · commit 2daacef5778c · 2025-01-11T03:35:35.000+11:00
* [ES|QL] Enable KQL function as a tech preview (#119730) (cherry picked from commit 31f11c3) # Conflicts: # docs/reference/esql/esql-limitations.asciidoc # server/src/main/java/org/elasticsearch/TransportVersions.java # server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java * Update server/src/main/java/org/elasticsearch/TransportVersions.java
diff --git a/docs/changelog/119730.yaml b/docs/changelog/119730.yaml
@@ -0,0 +1,5 @@
+pr: 119730
+summary: Enable KQL function as a tech preview
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/reference/esql/functions/kibana/definition/kql.json b/docs/reference/esql/functions/kibana/definition/kql.json
diff --git a/docs/reference/esql/functions/search-functions.asciidoc b/docs/reference/esql/functions/search-functions.asciidoc
@@ -16,9 +16,11 @@ See <<esql-limitations-full-text-search,full text search limitations>> for infor
 {esql} supports these full-text search functions:
 
 // tag::search_list[]
+* experimental:[] <<esql-kql>>
 * experimental:[] <<esql-match>>
 * experimental:[] <<esql-qstr>>
 // end::search_list[]
 
+include::layout/kql.asciidoc[]
 include::layout/match.asciidoc[]
 include::layout/qstr.asciidoc[]
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -158,6 +158,9 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_ENABLE_NODE_LEVEL_REDUCTION = def(8_818_00_0);
     public static final TransportVersion JINA_AI_INTEGRATION_ADDED = def(8_819_00_0);
     public static final TransportVersion TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC = def(8_820_00_0);
+    public static final TransportVersion REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX = def(8_821_00_0);
+    public static final TransportVersion ELASTIC_INFERENCE_SERVICE_UNIFIED_CHAT_COMPLETIONS_INTEGRATION = def(8_822_00_0);
+    public static final TransportVersion KQL_QUERY_TECH_PREVIEW = def(8_823_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
@@ -9,8 +9,6 @@
 
 package org.elasticsearch.rest.action.search;
 
-import org.elasticsearch.Build;
-
 import java.util.HashSet;
 import java.util.Set;
 
@@ -55,9 +53,7 @@ private SearchCapabilities() {}
         capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE);
         capabilities.add(MOVING_FN_RIGHT_MATH);
         capabilities.add(K_DEFAULT_TO_SIZE);
-        if (Build.current().isSnapshot()) {
-            capabilities.add(KQL_QUERY_SUPPORTED);
-        }
+        capabilities.add(KQL_QUERY_SUPPORTED);
         CAPABILITIES = Set.copyOf(capabilities);
     }
 }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java
@@ -15,10 +15,8 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.kql.KqlPlugin;
 import org.junit.Before;
-import org.junit.BeforeClass;
 
 import java.util.Collection;
 import java.util.List;
@@ -27,12 +25,6 @@
 import static org.hamcrest.CoreMatchers.containsString;
 
 public class KqlFunctionIT extends AbstractEsqlIntegTestCase {
-
-    @BeforeClass
-    protected static void ensureKqlFunctionEnabled() {
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-    }
-
     @Before
     public void setupIndex() {
         createAndPopulateIndex();
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -445,7 +445,7 @@ public enum Cap {
         /**
          * KQL function
          */
-        KQL_FUNCTION(Build.current().isSnapshot()),
+        KQL_FUNCTION,
 
         /**
          * Hash function
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -424,7 +424,10 @@ private static FunctionDefinition[][] functions() {
                 def(MvSum.class, MvSum::new, "mv_sum"),
                 def(Split.class, Split::new, "split") },
             // fulltext functions
-            new FunctionDefinition[] { def(Match.class, bi(Match::new), "match"), def(QueryString.class, uni(QueryString::new), "qstr") } };
+            new FunctionDefinition[] {
+                def(Kql.class, uni(Kql::new), "kql"),
+                def(Match.class, bi(Match::new), "match"),
+                def(QueryString.class, uni(QueryString::new), "qstr") } };
 
     }
 
@@ -434,7 +437,6 @@ private static FunctionDefinition[][] snapshotFunctions() {
                 // The delay() function is for debug/snapshot environments only and should never be enabled in a non-snapshot build.
                 // This is an experimental function and can be removed without notice.
                 def(Delay.class, Delay::new, "delay"),
-                def(Kql.class, uni(Kql::new), "kql"),
                 def(Rate.class, Rate::withUnresolvedTimestamp, "rate"),
                 def(Term.class, bi(Term::new), "term") } };
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java
@@ -25,10 +25,8 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(MultiMatchQueryPredicate.ENTRY);
         entries.add(QueryString.ENTRY);
         entries.add(Match.ENTRY);
+        entries.add(Kql.ENTRY);
 
-        if (EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()) {
-            entries.add(Kql.ENTRY);
-        }
         if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) {
             entries.add(Term.ENTRY);
         }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
@@ -1261,9 +1261,6 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception {
     }
 
     public void testKqlFunctionsNotAllowedAfterCommands() throws Exception {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         // Source commands
         assertEquals("1:13: [KQL] function cannot be used after SHOW", error("show info | where kql(\"8.16.0\")"));
         assertEquals("1:17: [KQL] function cannot be used after ROW", error("row a= \"Anna\" | where kql(\"Anna\")"));
@@ -1321,9 +1318,6 @@ public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception {
     }
 
     public void testKqlFunctionOnlyAllowedInWhere() throws Exception {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         assertEquals("1:9: [KQL] function is only supported in WHERE commands", error("row a = kql(\"Anna\")"));
         checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function");
     }
@@ -1375,9 +1369,6 @@ public void testQueryStringFunctionArgNotNullOrConstant() throws Exception {
     }
 
     public void testKqlFunctionArgNotNullOrConstant() throws Exception {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         assertEquals(
             "1:19: argument of [kql(first_name)] must be a constant, received [first_name]",
             error("from test | where kql(first_name)")
@@ -1391,9 +1382,6 @@ public void testQueryStringWithDisjunctions() {
     }
 
     public void testKqlFunctionWithDisjunctions() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function");
     }
 
@@ -1436,8 +1424,6 @@ public void testFullTextFunctionsDisjunctions() {
         checkWithFullTextFunctionsDisjunctions("MATCH", "match(last_name, \"Smith\")", "function");
         checkWithFullTextFunctionsDisjunctions(":", "last_name : \"Smith\"", "operator");
         checkWithFullTextFunctionsDisjunctions("QSTR", "qstr(\"last_name: Smith\")", "function");
-
-        assumeTrue("KQL function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
         checkWithFullTextFunctionsDisjunctions("KQL", "kql(\"last_name: Smith\")", "function");
     }
 
@@ -1466,9 +1452,6 @@ public void testQueryStringFunctionWithNonBooleanFunctions() {
     }
 
     public void testKqlFunctionWithNonBooleanFunctions() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function");
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java
@@ -10,21 +10,14 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.junit.BeforeClass;
 
 import java.util.List;
 import java.util.function.Supplier;
 
 public class KqlTests extends NoneFieldFullTextFunctionTestCase {
-    @BeforeClass
-    protected static void ensureKqlFunctionEnabled() {
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-    }
-
     public KqlTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
         super(testCaseSupplier);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
@@ -791,9 +791,6 @@ public void testMatchFunctionMultipleMatchClauses() {
      *       \_EsQueryExec[test], indexMode[standard], query[{"kql":{"query":"last_name: Smith"}}]
      */
     public void testKqlFunction() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         var plan = plannerOptimizer.plan("""
             from test
             | where kql("last_name: Smith")
@@ -822,9 +819,6 @@ public void testKqlFunction() {
      *        "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324]
      */
     public void testKqlFunctionConjunctionWhereOperands() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith") and emp_no > 10010
@@ -859,9 +853,6 @@ public void testKqlFunctionConjunctionWhereOperands() {
      *       "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354]
      */
     public void testKqlFunctionWithFunctionsPushedToLucene() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith") and cidr_match(ip, "127.0.0.1/32")
@@ -897,9 +888,6 @@ public void testKqlFunctionWithFunctionsPushedToLucene() {
      *       "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324]
      */
     public void testKqlFunctionMultipleWhereClauses() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith")
@@ -934,9 +922,6 @@ public void testKqlFunctionMultipleWhereClauses() {
      *       {"kql":{"query":"emp_no > 10010"}}],"boost":1.0}}]
      */
     public void testKqlFunctionMultipleKqlClauses() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith") and kql("emp_no > 10010")
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.kql;
 
-import org.elasticsearch.Build;
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
@@ -18,10 +17,6 @@
 public class KqlPlugin extends Plugin implements SearchPlugin, ExtensiblePlugin {
     @Override
     public List<QuerySpec<?>> getQueries() {
-        if (Build.current().isSnapshot()) {
-            return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent));
-        }
-
-        return List.of();
+        return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent));
     }
 }
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java
@@ -97,7 +97,7 @@ public static KqlQueryBuilder fromXContent(XContentParser parser) {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.KQL_QUERY_ADDED;
+        return TransportVersions.KQL_QUERY_TECH_PREVIEW;
     }
 
     public String queryString() {
diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.kql.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Build;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -22,7 +21,6 @@
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.xpack.kql.KqlPlugin;
 import org.hamcrest.Matchers;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -36,12 +34,6 @@
 import static org.hamcrest.Matchers.nullValue;
 
 public class KqlQueryBuilderTests extends AbstractQueryTestCase<KqlQueryBuilder> {
-    @BeforeClass
-    protected static void ensureSnapshotBuild() {
-        assumeTrue("requires snapshot builds", Build.current().isSnapshot());
-    }
-
-    @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return List.of(KqlPlugin.class);
     }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml
@@ -163,4 +163,4 @@ setup:
   - match: {esql.functions.cos: $functions_cos}
   - gt: {esql.functions.to_long: $functions_to_long}
   - match: {esql.functions.coalesce: $functions_coalesce}
-  - length: {esql.functions: 129} # check the "sister" test above for a likely update to the same esql.functions length check
+  - length: {esql.functions: 130} # check the "sister" test above for a likely update to the same esql.functions length check

Original file line number	Diff line number	Diff line change
`@@ -1261,9 +1261,6 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception {`
`1261`	`1261`	`}`
`1262`	`1262`
`1263`	`1263`	`public void testKqlFunctionsNotAllowedAfterCommands() throws Exception {`
`1264`		`- // Skip test if the kql function is not enabled.`
`1265`		`- assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());`
`1266`		`-`
`1267`	`1264`	`// Source commands`
`1268`	`1265`	`assertEquals("1:13: [KQL] function cannot be used after SHOW", error("show info \| where kql(\"8.16.0\")"));`
`1269`	`1266`	`assertEquals("1:17: [KQL] function cannot be used after ROW", error("row a= \"Anna\" \| where kql(\"Anna\")"));`
`@@ -1321,9 +1318,6 @@ public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception {`
`1321`	`1318`	`}`
`1322`	`1319`
`1323`	`1320`	`public void testKqlFunctionOnlyAllowedInWhere() throws Exception {`
`1324`		`- // Skip test if the kql function is not enabled.`
`1325`		`- assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());`
`1326`		`-`
`1327`	`1321`	`assertEquals("1:9: [KQL] function is only supported in WHERE commands", error("row a = kql(\"Anna\")"));`
`1328`	`1322`	`checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function");`
`1329`	`1323`	`}`
`@@ -1375,9 +1369,6 @@ public void testQueryStringFunctionArgNotNullOrConstant() throws Exception {`
`1375`	`1369`	`}`
`1376`	`1370`
`1377`	`1371`	`public void testKqlFunctionArgNotNullOrConstant() throws Exception {`
`1378`		`- // Skip test if the kql function is not enabled.`
`1379`		`- assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());`
`1380`		`-`
`1381`	`1372`	`assertEquals(`
`1382`	`1373`	`"1:19: argument of [kql(first_name)] must be a constant, received [first_name]",`
`1383`	`1374`	`error("from test \| where kql(first_name)")`
`@@ -1391,9 +1382,6 @@ public void testQueryStringWithDisjunctions() {`
`1391`	`1382`	`}`
`1392`	`1383`
`1393`	`1384`	`public void testKqlFunctionWithDisjunctions() {`
`1394`		`- // Skip test if the kql function is not enabled.`
`1395`		`- assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());`
`1396`		`-`
`1397`	`1385`	`checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function");`
`1398`	`1386`	`}`
`1399`	`1387`
`@@ -1436,8 +1424,6 @@ public void testFullTextFunctionsDisjunctions() {`
`1436`	`1424`	`checkWithFullTextFunctionsDisjunctions("MATCH", "match(last_name, \"Smith\")", "function");`
`1437`	`1425`	`checkWithFullTextFunctionsDisjunctions(":", "last_name : \"Smith\"", "operator");`
`1438`	`1426`	`checkWithFullTextFunctionsDisjunctions("QSTR", "qstr(\"last_name: Smith\")", "function");`
`1439`		`-`
`1440`		`- assumeTrue("KQL function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());`
`1441`	`1427`	`checkWithFullTextFunctionsDisjunctions("KQL", "kql(\"last_name: Smith\")", "function");`
`1442`	`1428`	`}`
`1443`	`1429`
`@@ -1466,9 +1452,6 @@ public void testQueryStringFunctionWithNonBooleanFunctions() {`
`1466`	`1452`	`}`
`1467`	`1453`
`1468`	`1454`	`public void testKqlFunctionWithNonBooleanFunctions() {`
`1469`		`- // Skip test if the kql function is not enabled.`
`1470`		`- assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());`
`1471`		`-`
`1472`	`1455`	`checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function");`
`1473`	`1456`	`}`
`1474`	`1457`