From 28c00b294a96ff42a13f8ee7cf61199635f514e0 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Fri, 30 May 2025 14:36:14 +0200 Subject: [PATCH 01/14] Refactor verifier tests --- .../xpack/esql/analysis/VerifierTests.java | 378 ++++-------------- 1 file changed, 86 insertions(+), 292 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index d76a355a6c9a9..d117206281628 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1259,112 +1259,83 @@ public void testMatchFunctionIsNotNullable() { } public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { - // Source commands - assertEquals("1:13: [QSTR] function cannot be used after SHOW", error("show info | where qstr(\"8.16.0\")")); - assertEquals("1:17: [QSTR] function cannot be used after ROW", error("row a= \"Anna\" | where qstr(\"Anna\")")); + testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("QSTR", "qstr(\"field_name: Anna\")"); + } - // Processing commands - assertEquals( - "1:43: [QSTR] function cannot be used after DISSECT", - error("from test | dissect first_name \"%{foo}\" | where qstr(\"Connection\")") - ); - assertEquals("1:27: [QSTR] function cannot be used after DROP", error("from test | drop emp_no | where qstr(\"Anna\")")); - assertEquals( - "1:71: [QSTR] function cannot be used after ENRICH", - error("from test | enrich languages on languages with lang = language_name | where qstr(\"Anna\")") - ); - assertEquals("1:26: [QSTR] function cannot be used after EVAL", error("from test | eval z = 2 | where qstr(\"Anna\")")); - assertEquals( - "1:44: [QSTR] function cannot be used after GROK", - error("from test | grok last_name \"%{WORD:foo}\" | where qstr(\"Anna\")") - ); - assertEquals("1:27: [QSTR] function cannot be used after KEEP", error("from test | keep emp_no | where qstr(\"Anna\")")); - assertEquals("1:24: [QSTR] function cannot be used after LIMIT", error("from test | limit 10 | where qstr(\"Anna\")")); - assertEquals( - "1:35: [QSTR] function cannot be used after MV_EXPAND", - error("from test | mv_expand last_name | where qstr(\"Anna\")") - ); - assertEquals( - "1:45: [QSTR] function cannot be used after RENAME", - error("from test | rename last_name as full_name | where qstr(\"Anna\")") - ); + public void testKqlFunctionsNotAllowedAfterCommands() throws Exception { + testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("KQL", "kql(\"field_name: Anna\")"); + } + + public void testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands(String functionName, String functionInvocation) throws Exception { + // Source commands + assertEquals("1:13: [" + functionName + "] function cannot be used after SHOW", error("show info | where " + functionInvocation)); assertEquals( - "1:52: [QSTR] function cannot be used after STATS", - error("from test | STATS c = COUNT(emp_no) BY languages | where qstr(\"Anna\")") + "1:17: [" + functionName + "] function cannot be used after ROW", + error("row a= \"Anna\" | where " + functionInvocation) ); - // Some combination of processing commands + // Processing commands assertEquals( - "1:38: [QSTR] function cannot be used after LIMIT", - error("from test | keep emp_no | limit 10 | where qstr(\"Anna\")") + "1:43: [" + functionName + "] function cannot be used after DISSECT", + error("from test | dissect first_name \"%{foo}\" | where " + functionInvocation) ); assertEquals( - "1:46: [QSTR] function cannot be used after MV_EXPAND", - error("from test | limit 10 | mv_expand last_name | where qstr(\"Anna\")") + "1:27: [" + functionName + "] function cannot be used after DROP", + error("from test | drop emp_no | where " + functionInvocation) ); assertEquals( - "1:52: [QSTR] function cannot be used after KEEP", - error("from test | mv_expand last_name | keep last_name | where qstr(\"Anna\")") + "1:71: [" + functionName + "] function cannot be used after ENRICH", + error("from test | enrich languages on languages with lang = language_name | where " + functionInvocation) ); assertEquals( - "1:77: [QSTR] function cannot be used after RENAME", - error("from test | STATS c = COUNT(emp_no) BY languages | rename c as total_emps | where qstr(\"Anna\")") + "1:26: [" + functionName + "] function cannot be used after EVAL", + error("from test | eval z = 2 | where " + functionInvocation) ); assertEquals( - "1:54: [QSTR] function cannot be used after KEEP", - error("from test | rename last_name as name | keep emp_no | where qstr(\"Anna\")") + "1:44: [" + functionName + "] function cannot be used after GROK", + error("from test | grok last_name \"%{WORD:foo}\" | where " + functionInvocation) ); - } - - public void testKqlFunctionsNotAllowedAfterCommands() throws Exception { - // Source commands - assertEquals("1:13: [KQL] function cannot be used after SHOW", error("show info | where kql(\"8.16.0\")")); - assertEquals("1:17: [KQL] function cannot be used after ROW", error("row a= \"Anna\" | where kql(\"Anna\")")); - - // Processing commands assertEquals( - "1:43: [KQL] function cannot be used after DISSECT", - error("from test | dissect first_name \"%{foo}\" | where kql(\"Connection\")") + "1:27: [" + functionName + "] function cannot be used after KEEP", + error("from test | keep emp_no | where " + functionInvocation) ); - assertEquals("1:27: [KQL] function cannot be used after DROP", error("from test | drop emp_no | where kql(\"Anna\")")); assertEquals( - "1:71: [KQL] function cannot be used after ENRICH", - error("from test | enrich languages on languages with lang = language_name | where kql(\"Anna\")") + "1:24: [" + functionName + "] function cannot be used after LIMIT", + error("from test | limit 10 | where " + functionInvocation) ); - assertEquals("1:26: [KQL] function cannot be used after EVAL", error("from test | eval z = 2 | where kql(\"Anna\")")); assertEquals( - "1:44: [KQL] function cannot be used after GROK", - error("from test | grok last_name \"%{WORD:foo}\" | where kql(\"Anna\")") + "1:35: [" + functionName + "] function cannot be used after MV_EXPAND", + error("from test | mv_expand last_name | where " + functionInvocation) ); - assertEquals("1:27: [KQL] function cannot be used after KEEP", error("from test | keep emp_no | where kql(\"Anna\")")); - assertEquals("1:24: [KQL] function cannot be used after LIMIT", error("from test | limit 10 | where kql(\"Anna\")")); - assertEquals("1:35: [KQL] function cannot be used after MV_EXPAND", error("from test | mv_expand last_name | where kql(\"Anna\")")); assertEquals( - "1:45: [KQL] function cannot be used after RENAME", - error("from test | rename last_name as full_name | where kql(\"Anna\")") + "1:45: [" + functionName + "] function cannot be used after RENAME", + error("from test | rename last_name as full_name | where " + functionInvocation) ); assertEquals( - "1:52: [KQL] function cannot be used after STATS", - error("from test | STATS c = COUNT(emp_no) BY languages | where kql(\"Anna\")") + "1:52: [" + functionName + "] function cannot be used after STATS", + error("from test | STATS c = COUNT(emp_no) BY languages | where " + functionInvocation) ); // Some combination of processing commands - assertEquals("1:38: [KQL] function cannot be used after LIMIT", error("from test | keep emp_no | limit 10 | where kql(\"Anna\")")); assertEquals( - "1:46: [KQL] function cannot be used after MV_EXPAND", - error("from test | limit 10 | mv_expand last_name | where kql(\"Anna\")") + "1:38: [" + functionName + "] function cannot be used after LIMIT", + error("from test | keep emp_no | limit 10 | where " + functionInvocation) + ); + assertEquals( + "1:46: [" + functionName + "] function cannot be used after MV_EXPAND", + error("from test | limit 10 | mv_expand last_name | where " + functionInvocation) ); assertEquals( - "1:52: [KQL] function cannot be used after KEEP", - error("from test | mv_expand last_name | keep last_name | where kql(\"Anna\")") + "1:52: [" + functionName + "] function cannot be used after KEEP", + error("from test | mv_expand last_name | keep last_name | where " + functionInvocation) ); assertEquals( - "1:77: [KQL] function cannot be used after RENAME", - error("from test | STATS c = COUNT(emp_no) BY languages | rename c as total_emps | where kql(\"Anna\")") + "1:77: [" + functionName + "] function cannot be used after RENAME", + error("from test | STATS c = COUNT(emp_no) BY languages | rename c as total_emps | where " + functionInvocation) ); assertEquals( - "1:54: [KQL] function cannot be used after DROP", - error("from test | rename last_name as name | drop emp_no | where kql(\"Anna\")") + "1:54: [" + functionName + "] function cannot be used after DROP", + error("from test | rename last_name as name | drop emp_no | where " + functionInvocation) ); } @@ -1425,25 +1396,17 @@ public void testKqlFunctionArgNotNullOrConstant() throws Exception { // Other value types are tested in KqlFunctionTests } - public void testQueryStringWithDisjunctions() { - checkWithDisjunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); - } - - public void testKqlFunctionWithDisjunctions() { - checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function"); - } - - public void testMatchFunctionWithDisjunctions() { + public void testFullTextFunctionsWithDisjunctions() { checkWithDisjunctions("MATCH", "match(first_name, \"Anna\")", "function"); - } - - public void testTermFunctionWithDisjunctions() { - assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); - checkWithDisjunctions("Term", "term(first_name, \"Anna\")", "function"); - } - - public void testMatchOperatorWithDisjunctions() { checkWithDisjunctions(":", "first_name : \"Anna\"", "operator"); + checkWithDisjunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); + checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function"); + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + checkWithDisjunctions("Term", "term(first_name, \"Anna\")", "function"); + } + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + checkWithDisjunctions("MultiMatch", "multi_match(\"Anna\", first_name)", "function"); + } } private void checkWithDisjunctions(String functionName, String functionInvocation, String functionType) { @@ -1462,6 +1425,9 @@ public void testFullTextFunctionsDisjunctions() { checkWithFullTextFunctionsDisjunctions("last_name : \"Smith\""); checkWithFullTextFunctionsDisjunctions("qstr(\"last_name: Smith\")"); checkWithFullTextFunctionsDisjunctions("kql(\"last_name: Smith\")"); + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + checkWithFullTextFunctionsDisjunctions("term(last_name, \"Smith\")"); + } } private void checkWithFullTextFunctionsDisjunctions(String functionInvocation) { @@ -2173,195 +2139,35 @@ public void testLookupJoinDataTypeMismatch() { } public void testMatchOptions() { - // Check positive cases - query("FROM test | WHERE match(first_name, \"Jean\", {\"analyzer\": \"standard\"})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"boost\": 2.1})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"fuzziness\": 2})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"fuzziness\": \"AUTO\"})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"fuzzy_transpositions\": false})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"lenient\": false})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"max_expansions\": 10})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"minimum_should_match\": \"2\"})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"operator\": \"AND\"})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"prefix_length\": 2})"); - query("FROM test | WHERE match(first_name, \"Jean\", {\"auto_generate_synonyms_phrase_query\": true})"); - - // Check all data types for available options - DataType[] optionTypes = new DataType[] { INTEGER, LONG, FLOAT, DOUBLE, KEYWORD, BOOLEAN }; - for (Map.Entry allowedOptions : Match.ALLOWED_OPTIONS.entrySet()) { - String optionName = allowedOptions.getKey(); - DataType optionType = allowedOptions.getValue(); - // Check every possible type for the option - we'll try to convert it to the expected type - for (DataType currentType : optionTypes) { - String optionValue = switch (currentType) { - case BOOLEAN -> String.valueOf(randomBoolean()); - case INTEGER -> String.valueOf(randomIntBetween(0, 100000)); - case LONG -> String.valueOf(randomLong()); - case FLOAT -> String.valueOf(randomFloat()); - case DOUBLE -> String.valueOf(randomDouble()); - case KEYWORD -> randomAlphaOfLength(10); - default -> throw new IllegalArgumentException("Unsupported option type: " + currentType); - }; - String queryOptionValue = optionValue; - if (currentType == KEYWORD) { - queryOptionValue = "\"" + optionValue + "\""; - } - - String query = "FROM test | WHERE match(first_name, \"Jean\", {\"" + optionName + "\": " + queryOptionValue + "})"; - try { - // Check conversion is possible - DataTypeConverter.convert(optionValue, optionType); - // If no exception was thrown, conversion is possible and should be done - query(query); - } catch (InvalidArgumentException e) { - // Conversion is not possible, query should fail - assertEquals( - "1:19: Invalid option [" - + optionName - + "] in [match(first_name, \"Jean\", {\"" - + optionName - + "\": " - + queryOptionValue - + "})], cannot cast [" - + optionValue - + "] to [" - + optionType.typeName() - + "]", - error(query) - ); - } - } - } + checkOptionDataTypes(Match.ALLOWED_OPTIONS, "FROM test | WHERE match(first_name, \"Jean\", {\"%s\": %s})"); + } - assertThat( - error("FROM test | WHERE match(first_name, \"Jean\", {\"unknown_option\": true})"), - containsString( - "1:19: Invalid option [unknown_option] in [match(first_name, \"Jean\", {\"unknown_option\": true})]," + " expected one of " - ) - ); + public void testMultiMatchOptions() { + checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"%s\": %s})"); } public void testQueryStringOptions() { - // Check positive cases - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"analyzer\": \"standard\"})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"allow_leading_wildcard\": false})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"analyze_wildcard\": false})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"auto_generate_synonyms_phrase_query\": true})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"boost\": 2.1})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"default_field\": \"field1\"})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"default_operator\": \"AND\"})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"enable_position_increments\": false})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"fuzziness\": 2})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"fuzziness\": \"AUTO\"})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"fuzzy_prefix_length\": 5})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"fuzzy_transpositions\": false})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"lenient\": false})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"max_determinized_states\": 10})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"minimum_should_match\": \"2\"})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"quote_analyzer\": \"qnalyzer_1\"})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"quote_field_suffix\": \"q_suffix\"})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"phrase_slop\": 10})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"rewrite\": \"r1\"})"); - query("FROM test | WHERE QSTR(\"first_name: Jean\", {\"time_zone\": \"time_zone\"})"); - - // Check all data types for available options - DataType[] optionTypes = new DataType[] { INTEGER, LONG, FLOAT, DOUBLE, KEYWORD, BOOLEAN }; - for (Map.Entry allowedOptions : QueryString.ALLOWED_OPTIONS.entrySet()) { - String optionName = allowedOptions.getKey(); - DataType optionType = allowedOptions.getValue(); - // Check every possible type for the option - we'll try to convert it to the expected type - for (DataType currentType : optionTypes) { - String optionValue = switch (currentType) { - case BOOLEAN -> String.valueOf(randomBoolean()); - case INTEGER -> String.valueOf(randomIntBetween(0, 100000)); - case LONG -> String.valueOf(randomLong()); - case FLOAT -> String.valueOf(randomFloat()); - case DOUBLE -> String.valueOf(randomDouble()); - case KEYWORD -> randomAlphaOfLength(10); - default -> throw new IllegalArgumentException("Unsupported option type: " + currentType); - }; - String queryOptionValue = optionValue; - if (currentType == KEYWORD) { - queryOptionValue = "\"" + optionValue + "\""; - } - - String query = "FROM test | WHERE QSTR(\"first_name: Jean\", {\"" + optionName + "\": " + queryOptionValue + "})"; - try { - // Check conversion is possible - DataTypeConverter.convert(optionValue, optionType); - // If no exception was thrown, conversion is possible and should be done - query(query); - } catch (InvalidArgumentException e) { - // Conversion is not possible, query should fail - assertEquals( - "1:19: Invalid option [" - + optionName - + "] in [QSTR(\"first_name: Jean\", {\"" - + optionName - + "\": " - + queryOptionValue - + "})], cannot cast [" - + optionValue - + "] to [" - + optionType.typeName() - + "]", - error(query) - ); - } - } - } - - assertThat( - error("FROM test | WHERE QSTR(\"first_name: Jean\", {\"unknown_option\": true})"), - containsString( - "1:20: Invalid option [unknown_option] in [QSTR(\"first_name: Jean\", {\"unknown_option\": true})]," + " expected one of " - ) - ); + checkOptionDataTypes(QueryString.ALLOWED_OPTIONS, "FROM test | WHERE QSTR(\"first_name: Jean\", {\"%s\": %s})"); } - public void testMultiMatchOptions() { - // Check positive cases - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name)"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, {\"analyzer\": \"standard\"})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"analyzer\": \"standard\"})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"slop\": 10})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"auto_generate_synonyms_phrase_query\": true})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"fuzziness\": 2})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"fuzzy_transpositions\": false})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"lenient\": false})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"max_expansions\": 10})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"minimum_should_match\": \"2\"})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"operator\": \"AND\"})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"prefix_length\": 2})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"tie_breaker\": 1.0})"); - query("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"type\": \"best_fields\"})"); - - // Check all data types for available options + /** + * Check all data types for available options. When conversion is not possible, checks that it's an error + */ + private void checkOptionDataTypes(Map allowedOptionsMap, String queryTemplate) { DataType[] optionTypes = new DataType[] { INTEGER, LONG, FLOAT, DOUBLE, KEYWORD, BOOLEAN }; - for (Map.Entry allowedOptions : MultiMatch.OPTIONS.entrySet()) { + for (Map.Entry allowedOptions : allowedOptionsMap.entrySet()) { String optionName = allowedOptions.getKey(); DataType optionType = allowedOptions.getValue(); + // Check every possible type for the option - we'll try to convert it to the expected type for (DataType currentType : optionTypes) { - String optionValue = switch (currentType) { - case BOOLEAN -> String.valueOf(randomBoolean()); - case INTEGER -> String.valueOf(randomIntBetween(0, 100000)); - case LONG -> String.valueOf(randomLong()); - case FLOAT -> String.valueOf(randomFloat()); - case DOUBLE -> String.valueOf(randomDouble()); - case KEYWORD -> randomAlphaOfLength(10); - default -> throw new IllegalArgumentException("Unsupported option type: " + currentType); - }; + String optionValue = exampleValueForType(currentType); String queryOptionValue = optionValue; if (currentType == KEYWORD) { queryOptionValue = "\"" + optionValue + "\""; } - String query = "FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"" - + optionName - + "\": " - + queryOptionValue - + "})"; + String query = String.format(Locale.ROOT, queryTemplate, optionName, queryOptionValue); try { // Check conversion is possible DataTypeConverter.convert(optionValue, optionType); @@ -2369,35 +2175,27 @@ public void testMultiMatchOptions() { query(query); } catch (InvalidArgumentException e) { // Conversion is not possible, query should fail - assertEquals( - "1:19: Invalid option [" - + optionName - + "] in [MULTI_MATCH(\"Jean\", first_name, last_name, {\"" - + optionName - + "\": " - + queryOptionValue - + "})], cannot " - + (optionType == OBJECT ? "convert from" : "cast") - + " [" - + optionValue - + "]" - + (optionType == OBJECT ? (", type [keyword]") : "") - + " to [" - + optionType.typeName() - + "]", - error(query) - ); + String error = error(query); + assertThat(error, containsString("Invalid option [" + optionName + "]")); + assertThat(error, containsString("cannot cast [" + optionValue + "] to [" + optionType.typeName() + "]")); } } } - assertThat( - error("FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"unknown_option\": true})"), - containsString( - "1:19: Invalid option [unknown_option] in [MULTI_MATCH(\"Jean\", first_name, last_name, " - + "{\"unknown_option\": true})], expected one of " - ) - ); + String errorQuery = String.format(Locale.ROOT, queryTemplate, "unknown_option", "\"any_value\""); + assertThat(error(errorQuery), containsString("Invalid option [unknown_option]")); + } + + private static String exampleValueForType(DataType currentType) { + return switch (currentType) { + case BOOLEAN -> String.valueOf(randomBoolean()); + case INTEGER -> String.valueOf(randomIntBetween(0, 100000)); + case LONG -> String.valueOf(randomLong()); + case FLOAT -> String.valueOf(randomFloat()); + case DOUBLE -> String.valueOf(randomDouble()); + case KEYWORD -> randomAlphaOfLength(10); + default -> throw new IllegalArgumentException("Unsupported option type: " + currentType); + }; } public void testMultiMatchFunctionIsNotNullable() { @@ -2429,10 +2227,6 @@ public void testMultiMatchFunctionNotAllowedAfterCommands() throws Exception { ); } - public void testMultiMatchFunctionWithDisjunctions() { - checkWithDisjunctions("MultiMatch", "multi_match(\"Anna\", first_name, last_name)", "function"); - } - public void testMultiMatchFunctionWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions("MultiMatch", "multi_match(\"Anna\", first_name, last_name)", "function"); } From cd7a468a6cdbff29ba124b5d72c14472ba44514e Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Fri, 30 May 2025 15:03:35 +0200 Subject: [PATCH 02/14] Refactor verifier tests --- .../xpack/esql/analysis/VerifierTests.java | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index d117206281628..954df1d547de1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1239,22 +1239,31 @@ public void testMatchFunctionAndOperatorHaveCorrectErrorMessages() throws Except assertEquals("1:24: [:] operator cannot be used after LIMIT", error("from test | limit 10 | where first_name : \"Anna\"")); } + public void testFieldBasedFullTextFunctions() { + testFieldBasedWithNonIndexedColumn("MATCH", " match(text, \"cat\")", "function"); + testFieldBasedWithNonIndexedColumn(":", " text : \"cat\"", "operator"); + testFieldBasedWithNonIndexedColumn("MultiMatch", " multi_match(\"cat\", text)", "function"); + } + // These should pass eventually once we lift some restrictions on match function - public void testMatchWithNonIndexedColumnCurrentlyUnsupported() { - assertEquals( - "1:67: [MATCH] function cannot operate on [initial], which is not a field from an index mapping", - error("from test | eval initial = substring(first_name, 1) | where match(initial, \"A\")") + public void testFieldBasedWithNonIndexedColumn(String functionName, String functionInvocation, String functionType) { + assertThat( + error("from test | eval text = substring(first_name, 1) | where" + functionInvocation), + containsString("[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping") ); - assertEquals( - "1:67: [MATCH] function cannot operate on [text], which is not a field from an index mapping", - error("from test | eval text=concat(first_name, last_name) | where match(text, \"cat\")") + assertThat( + error("from test | eval text=concat(first_name, last_name) | where" + functionInvocation), + containsString("[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping") ); - } - - public void testMatchFunctionIsNotNullable() { - assertEquals( - "1:48: [MATCH] function cannot operate on [text::keyword], which is not a field from an index mapping", - error("row n = null | eval text = n + 5 | where match(text::keyword, \"Anna\")") + var keywordInvocation = functionInvocation.replace("text", "text::keyword"); + String keywordError = error("row n = null | eval text = n + 5 | where " + keywordInvocation); + assertThat( + keywordError, + containsString("[" + functionName + "] " + functionType + " cannot operate on") + ); + assertThat( + keywordError, + containsString("which is not a field from an index mapping") ); } From c417c010f8128c3546c68c5acf45cf5925116797 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Mon, 2 Jun 2025 17:00:53 +0200 Subject: [PATCH 03/14] Simplify tests --- .../function/fulltext/FullTextFunction.java | 18 +------- .../xpack/esql/analysis/VerifierTests.java | 45 ++++++++----------- 2 files changed, 20 insertions(+), 43 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 3abdebf63c1fc..19402be5fe733 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -204,23 +204,7 @@ private static void checkFullTextQueryFunctions(LogicalPlan plan, Failures failu checkCommandsBeforeExpression( plan, condition, - Match.class, - lp -> (lp instanceof Limit == false) && (lp instanceof Aggregate == false), - m -> "[" + m.functionName() + "] " + m.functionType(), - failures - ); - checkCommandsBeforeExpression( - plan, - condition, - MultiMatch.class, - lp -> (lp instanceof Limit == false) && (lp instanceof Aggregate == false), - m -> "[" + m.functionName() + "] " + m.functionType(), - failures - ); - checkCommandsBeforeExpression( - plan, - condition, - Term.class, + FullTextFunction.class, lp -> (lp instanceof Limit == false) && (lp instanceof Aggregate == false), m -> "[" + m.functionName() + "] " + m.functionType(), failures diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 954df1d547de1..ec1d87bd8ba41 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1215,44 +1215,37 @@ public void testMatchInsideEval() throws Exception { ); } - public void testMatchFunctionNotAllowedAfterCommands() throws Exception { - assertEquals( - "1:24: [MATCH] function cannot be used after LIMIT", - error("from test | limit 10 | where match(first_name, \"Anna\")") - ); - assertEquals( - "1:47: [MATCH] function cannot be used after STATS", - error("from test | STATS c = AVG(salary) BY gender | where match(gender, \"F\")") - ); + public void testFieldBasedFullTextFunctions() throws Exception { + testFieldBasedWithNonIndexedColumn("MATCH", "match(text, \"cat\")", "function"); + testFieldBasedWithNonIndexedColumn(":", "text : \"cat\"", "operator"); + testFieldBasedWithNonIndexedColumn("MultiMatch", "multi_match(\"cat\", text)", "function"); + + testFieldBasedFunctionNotAllowedAfterCommands("MATCH", "function", "match(first_name, \"Anna\")"); + testFieldBasedFunctionNotAllowedAfterCommands(":", "operator", "first_name : \"Anna\""); + testFieldBasedFunctionNotAllowedAfterCommands("MultiMatch", "function", "multi_match(\"Anna\", first_name)"); + testFieldBasedFunctionNotAllowedAfterCommands("KNN", "function", "knn(vector, [1, 2, 3])"); } - public void testMatchFunctionAndOperatorHaveCorrectErrorMessages() throws Exception { - assertEquals( - "1:24: [MATCH] function cannot be used after LIMIT", - error("from test | limit 10 | where match(first_name, \"Anna\")") + public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, String functionType, String functionInvocation) throws Exception { + assertThat( + error("from test | limit 10 | where " + functionInvocation), + containsString("[" + functionName + "] " + functionType + " cannot be used after LIMIT") ); - assertEquals( - "1:24: [MATCH] function cannot be used after LIMIT", - error("from test | limit 10 | where match ( first_name, \"Anna\" ) ") + String fieldName = "KNN".equals(functionName) ? "vector" : "first_name"; + assertThat( + error("from test | STATS c = COUNT(emp_no) BY " + fieldName + " | where " + functionInvocation), + containsString("[" + functionName + "] " + functionType + " cannot be used after STATS") ); - assertEquals("1:24: [:] operator cannot be used after LIMIT", error("from test | limit 10 | where first_name:\"Anna\"")); - assertEquals("1:24: [:] operator cannot be used after LIMIT", error("from test | limit 10 | where first_name : \"Anna\"")); - } - - public void testFieldBasedFullTextFunctions() { - testFieldBasedWithNonIndexedColumn("MATCH", " match(text, \"cat\")", "function"); - testFieldBasedWithNonIndexedColumn(":", " text : \"cat\"", "operator"); - testFieldBasedWithNonIndexedColumn("MultiMatch", " multi_match(\"cat\", text)", "function"); } // These should pass eventually once we lift some restrictions on match function public void testFieldBasedWithNonIndexedColumn(String functionName, String functionInvocation, String functionType) { assertThat( - error("from test | eval text = substring(first_name, 1) | where" + functionInvocation), + error("from test | eval text = substring(first_name, 1) | where " + functionInvocation), containsString("[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping") ); assertThat( - error("from test | eval text=concat(first_name, last_name) | where" + functionInvocation), + error("from test | eval text=concat(first_name, last_name) | where " + functionInvocation), containsString("[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping") ); var keywordInvocation = functionInvocation.replace("text", "text::keyword"); From ba53a4d81a3847cb88523f46fe60eaaf9122679d Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Mon, 2 Jun 2025 17:24:00 +0200 Subject: [PATCH 04/14] Simplify tests --- .../xpack/esql/analysis/VerifierTests.java | 175 +++++------------- 1 file changed, 49 insertions(+), 126 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index ec1d87bd8ba41..6d620b994db76 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1341,27 +1341,17 @@ public void testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands(String fun ); } - public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception { - assertEquals("1:9: [QSTR] function is only supported in WHERE and STATS commands", error("row a = qstr(\"Anna\")")); - checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")", "function"); - } - - public void testKqlFunctionOnlyAllowedInWhere() throws Exception { - assertEquals("1:9: [KQL] function is only supported in WHERE and STATS commands", error("row a = kql(\"Anna\")")); - checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function"); - } - - public void testMatchFunctionOnlyAllowedInWhere() throws Exception { + public void testFullTextFunctionsOnlyAllowedInWhere() throws Exception { checkFullTextFunctionsOnlyAllowedInWhere("MATCH", "match(first_name, \"Anna\")", "function"); - } - - public void testTermFunctionOnlyAllowedInWhere() throws Exception { - assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); - checkFullTextFunctionsOnlyAllowedInWhere("Term", "term(first_name, \"Anna\")", "function"); - } - - public void testMatchOperatornOnlyAllowedInWhere() throws Exception { checkFullTextFunctionsOnlyAllowedInWhere(":", "first_name:\"Anna\"", "operator"); + checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")", "function"); + checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function"); + if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { + checkFullTextFunctionsOnlyAllowedInWhere("KNN", "knn(vector, [1, 2, 3])", "function"); + } + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + checkFullTextFunctionsOnlyAllowedInWhere("Term", "term(first_name, \"Anna\")", "function"); + } } private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation, String functionType) @@ -1378,6 +1368,12 @@ private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, Strin "1:47: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", error("from test | stats max_salary = max(salary) by " + functionInvocation) ); + if( "KQL".equals(functionName) || "QSTR".equals(functionName)) { + assertEquals( + "1:9: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", + error("row a = " + functionInvocation) + ); + } } public void testQueryStringFunctionArgNotNullOrConstant() throws Exception { @@ -1398,29 +1394,6 @@ public void testKqlFunctionArgNotNullOrConstant() throws Exception { // Other value types are tested in KqlFunctionTests } - public void testFullTextFunctionsWithDisjunctions() { - checkWithDisjunctions("MATCH", "match(first_name, \"Anna\")", "function"); - checkWithDisjunctions(":", "first_name : \"Anna\"", "operator"); - checkWithDisjunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); - checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function"); - if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - checkWithDisjunctions("Term", "term(first_name, \"Anna\")", "function"); - } - if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - checkWithDisjunctions("MultiMatch", "multi_match(\"Anna\", first_name)", "function"); - } - } - - private void checkWithDisjunctions(String functionName, String functionInvocation, String functionType) { - query("from test | where " + functionInvocation + " or length(first_name) > 12"); - query( - "from test | where (" - + functionInvocation - + " or first_name is not null) or (length(first_name) > 12 and match(last_name, \"Smith\"))" - ); - query("from test | where " + functionInvocation + " or (last_name is not null and first_name is null)"); - } - public void testFullTextFunctionsDisjunctions() { checkWithFullTextFunctionsDisjunctions("match(last_name, \"Smith\")"); checkWithFullTextFunctionsDisjunctions("multi_match(\"Smith\", first_name, last_name)"); @@ -1472,25 +1445,20 @@ private void checkWithFullTextFunctionsDisjunctions(String functionInvocation) { } - public void testQueryStringFunctionWithNonBooleanFunctions() { - checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); - } - - public void testKqlFunctionWithNonBooleanFunctions() { - checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function"); - } - - public void testMatchFunctionWithNonBooleanFunctions() { + public void testFullTextFunctionsWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions("MATCH", "match(first_name, \"Anna\")", "function"); - } - - public void testTermFunctionWithNonBooleanFunctions() { - assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); - checkFullTextFunctionsWithNonBooleanFunctions("Term", "term(first_name, \"Anna\")", "function"); - } - - public void testMatchOperatorWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions(":", "first_name:\"Anna\"", "operator"); + checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); + checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function"); + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + checkFullTextFunctionsWithNonBooleanFunctions("MultiMatch", "multi_match(\"Anna\", first_name, last_name)", "function"); + } + if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { + checkFullTextFunctionsWithNonBooleanFunctions("KNN", "knn(vector, [1, 2, 3])", "function"); + } + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + checkFullTextFunctionsWithNonBooleanFunctions("Term", "term(first_name, \"Anna\")", "function"); + } } private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, String functionInvocation, String functionType) { @@ -1561,18 +1529,6 @@ public void testMatchFunctionArgNotConstant() throws Exception { // Other value types are tested in QueryStringFunctionTests } - // These should pass eventually once we lift some restrictions on match function - public void testMatchFunctionCurrentlyUnsupportedBehaviour() throws Exception { - assertEquals( - "1:68: Unknown column [first_name]", - error("from test | stats max_salary = max(salary) by emp_no | where match(first_name, \"Anna\")") - ); - assertEquals( - "1:62: Unknown column [first_name]", - error("from test | stats max_salary = max(salary) by emp_no | where first_name : \"Anna\"") - ); - } - public void testMatchFunctionNullArgs() throws Exception { assertEquals( "1:19: first argument of [match(null, \"query\")] cannot be null, received [null]", @@ -1602,15 +1558,6 @@ public void testTermFunctionArgNotConstant() throws Exception { // Other value types are tested in QueryStringFunctionTests } - // These should pass eventually once we lift some restrictions on match function - public void testTermFunctionCurrentlyUnsupportedBehaviour() throws Exception { - assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); - assertEquals( - "1:67: Unknown column [first_name]", - error("from test | stats max_salary = max(salary) by emp_no | where term(first_name, \"Anna\")") - ); - } - public void testTermFunctionNullArgs() throws Exception { assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); assertEquals( @@ -2140,16 +2087,15 @@ public void testLookupJoinDataTypeMismatch() { ); } - public void testMatchOptions() { + public void testFullTextFunctionOptions() { checkOptionDataTypes(Match.ALLOWED_OPTIONS, "FROM test | WHERE match(first_name, \"Jean\", {\"%s\": %s})"); - } - - public void testMultiMatchOptions() { - checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"%s\": %s})"); - } - - public void testQueryStringOptions() { checkOptionDataTypes(QueryString.ALLOWED_OPTIONS, "FROM test | WHERE QSTR(\"first_name: Jean\", {\"%s\": %s})"); + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()){ + checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"%s\": %s})"); + } + if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { + checkOptionDataTypes(Knn.ALLOWED_OPTIONS, "FROM test | WHERE KNN(vector, [0.1, 0.2, 0.3], {\"%s\": %s})"); + } } /** @@ -2200,39 +2146,6 @@ private static String exampleValueForType(DataType currentType) { }; } - public void testMultiMatchFunctionIsNotNullable() { - assertEquals( - "1:62: [MultiMatch] function cannot operate on [text::keyword], which is not a field from an index mapping", - error("row n = null | eval text = n + 5 | where multi_match(\"Anna\", text::keyword)") - ); - } - - public void testMultiMatchWithNonIndexedColumnCurrentlyUnsupported() { - assertEquals( - "1:78: [MultiMatch] function cannot operate on [initial], which is not a field from an index mapping", - error("from test | eval initial = substring(first_name, 1) | where multi_match(\"A\", initial)") - ); - assertEquals( - "1:80: [MultiMatch] function cannot operate on [text], which is not a field from an index mapping", - error("from test | eval text=concat(first_name, last_name) | where multi_match(\"cat\", text)") - ); - } - - public void testMultiMatchFunctionNotAllowedAfterCommands() throws Exception { - assertEquals( - "1:24: [MultiMatch] function cannot be used after LIMIT", - error("from test | limit 10 | where multi_match(\"Anna\", first_name)") - ); - assertEquals( - "1:47: [MultiMatch] function cannot be used after STATS", - error("from test | STATS c = AVG(salary) BY gender | where multi_match(\"F\", gender)") - ); - } - - public void testMultiMatchFunctionWithNonBooleanFunctions() { - checkFullTextFunctionsWithNonBooleanFunctions("MultiMatch", "multi_match(\"Anna\", first_name, last_name)", "function"); - } - public void testMultiMatchFunctionArgNotConstant() throws Exception { assertEquals( "1:19: second argument of [match(first_name, first_name)] must be a constant, received [first_name]", @@ -2242,14 +2155,24 @@ public void testMultiMatchFunctionArgNotConstant() throws Exception { "1:59: second argument of [match(first_name, query)] must be a constant, received [query]", error("from test | eval query = concat(\"first\", \" name\") | where match(first_name, query)") ); - // Other value types are tested in QueryStringFunctionTests } - // Should pass eventually once we lift some restrictions on the multi-match function. + // Should pass eventually once we lift some restrictions on full text search functions. public void testMultiMatchFunctionCurrentlyUnsupportedBehaviour() throws Exception { - assertEquals( - "1:82: Unknown column [first_name]\nline 1:94: Unknown column [last_name]", - error("from test | stats max_salary = max(salary) by emp_no | where multi_match(\"Anna\", first_name, last_name)") + testFullTextFunctionsCurrentlyUnsupportedBehaviour("match(first_name, \"Anna\")"); + testFullTextFunctionsCurrentlyUnsupportedBehaviour("first_name : \"Anna\""); + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + testFullTextFunctionsCurrentlyUnsupportedBehaviour("multi_match(\"Anna\", first_name)"); + } + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + testFullTextFunctionsCurrentlyUnsupportedBehaviour("term(first_name, \"Anna\")"); + } + } + + private void testFullTextFunctionsCurrentlyUnsupportedBehaviour(String functionInvocation) throws Exception { + assertThat( + error("from test | stats max_salary = max(salary) by emp_no | where " + functionInvocation), + containsString("Unknown column [first_name]") ); } From e35cb96aa840a9177a875b9a5df3181c9d0eb2b4 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Mon, 2 Jun 2025 17:50:16 +0200 Subject: [PATCH 05/14] Simplify tests --- .../function/fulltext/MultiMatch.java | 2 +- .../xpack/esql/analysis/VerifierTests.java | 181 +++++++----------- 2 files changed, 74 insertions(+), 109 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MultiMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MultiMatch.java index 5014263ba755b..2c398c7f6c6f1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MultiMatch.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MultiMatch.java @@ -379,7 +379,7 @@ public String functionName() { private TypeResolution resolveFields() { return fields.stream() .map( - (Expression field) -> isNotNull(field, sourceText(), FIRST).and( + (Expression field) -> isNotNull(field, sourceText(), SECOND).and( isType( field, FIELD_DATA_TYPES::contains, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 6d620b994db76..e7001d4134b79 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1226,7 +1226,8 @@ public void testFieldBasedFullTextFunctions() throws Exception { testFieldBasedFunctionNotAllowedAfterCommands("KNN", "function", "knn(vector, [1, 2, 3])"); } - public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, String functionType, String functionInvocation) throws Exception { + public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, String functionType, String functionInvocation) + throws Exception { assertThat( error("from test | limit 10 | where " + functionInvocation), containsString("[" + functionName + "] " + functionType + " cannot be used after LIMIT") @@ -1242,22 +1243,20 @@ public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, S public void testFieldBasedWithNonIndexedColumn(String functionName, String functionInvocation, String functionType) { assertThat( error("from test | eval text = substring(first_name, 1) | where " + functionInvocation), - containsString("[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping") + containsString( + "[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping" + ) ); assertThat( error("from test | eval text=concat(first_name, last_name) | where " + functionInvocation), - containsString("[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping") + containsString( + "[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping" + ) ); var keywordInvocation = functionInvocation.replace("text", "text::keyword"); String keywordError = error("row n = null | eval text = n + 5 | where " + keywordInvocation); - assertThat( - keywordError, - containsString("[" + functionName + "] " + functionType + " cannot operate on") - ); - assertThat( - keywordError, - containsString("which is not a field from an index mapping") - ); + assertThat(keywordError, containsString("[" + functionName + "] " + functionType + " cannot operate on")); + assertThat(keywordError, containsString("which is not a field from an index mapping")); } public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { @@ -1346,12 +1345,15 @@ public void testFullTextFunctionsOnlyAllowedInWhere() throws Exception { checkFullTextFunctionsOnlyAllowedInWhere(":", "first_name:\"Anna\"", "operator"); checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")", "function"); checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function"); - if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { - checkFullTextFunctionsOnlyAllowedInWhere("KNN", "knn(vector, [1, 2, 3])", "function"); - } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { checkFullTextFunctionsOnlyAllowedInWhere("Term", "term(first_name, \"Anna\")", "function"); } + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + checkFullTextFunctionsOnlyAllowedInWhere("MultiMatch", "multi_match(\"Anna\", first_name, last_name)", "function"); + } + if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { + checkFullTextFunctionsOnlyAllowedInWhere("KNN", "knn(vector, [1, 2, 3])", "function"); + } } private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation, String functionType) @@ -1368,7 +1370,7 @@ private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, Strin "1:47: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", error("from test | stats max_salary = max(salary) by " + functionInvocation) ); - if( "KQL".equals(functionName) || "QSTR".equals(functionName)) { + if ("KQL".equals(functionName) || "QSTR".equals(functionName)) { assertEquals( "1:9: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", error("row a = " + functionInvocation) @@ -1376,24 +1378,6 @@ private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, Strin } } - public void testQueryStringFunctionArgNotNullOrConstant() throws Exception { - assertEquals( - "1:19: first argument of [qstr(first_name)] must be a constant, received [first_name]", - error("from test | where qstr(first_name)") - ); - assertEquals("1:19: first argument of [qstr(null)] cannot be null, received [null]", error("from test | where qstr(null)")); - // Other value types are tested in QueryStringFunctionTests - } - - public void testKqlFunctionArgNotNullOrConstant() throws Exception { - assertEquals( - "1:19: argument of [kql(first_name)] must be a constant, received [first_name]", - error("from test | where kql(first_name)") - ); - assertEquals("1:19: argument of [kql(null)] cannot be null, received [null]", error("from test | where kql(null)")); - // Other value types are tested in KqlFunctionTests - } - public void testFullTextFunctionsDisjunctions() { checkWithFullTextFunctionsDisjunctions("match(last_name, \"Smith\")"); checkWithFullTextFunctionsDisjunctions("multi_match(\"Smith\", first_name, last_name)"); @@ -1517,62 +1501,22 @@ private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, ); } - public void testMatchFunctionArgNotConstant() throws Exception { - assertEquals( - "1:19: second argument of [match(first_name, first_name)] must be a constant, received [first_name]", - error("from test | where match(first_name, first_name)") - ); - assertEquals( - "1:59: second argument of [match(first_name, query)] must be a constant, received [query]", - error("from test | eval query = concat(\"first\", \" name\") | where match(first_name, query)") - ); - // Other value types are tested in QueryStringFunctionTests - } - - public void testMatchFunctionNullArgs() throws Exception { - assertEquals( - "1:19: first argument of [match(null, \"query\")] cannot be null, received [null]", - error("from test | where match(null, \"query\")") - ); - assertEquals( - "1:19: second argument of [match(first_name, null)] cannot be null, received [null]", - error("from test | where match(first_name, null)") - ); - } - - public void testMatchTargetsExistingField() throws Exception { - assertEquals("1:39: Unknown column [first_name]", error("from test | keep emp_no | where match(first_name, \"Anna\")")); - assertEquals("1:33: Unknown column [first_name]", error("from test | keep emp_no | where first_name : \"Anna\"")); - } - - public void testTermFunctionArgNotConstant() throws Exception { - assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); - assertEquals( - "1:19: second argument of [term(first_name, first_name)] must be a constant, received [first_name]", - error("from test | where term(first_name, first_name)") - ); - assertEquals( - "1:59: second argument of [term(first_name, query)] must be a constant, received [query]", - error("from test | eval query = concat(\"first\", \" name\") | where term(first_name, query)") - ); - // Other value types are tested in QueryStringFunctionTests - } - - public void testTermFunctionNullArgs() throws Exception { - assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); - assertEquals( - "1:19: first argument of [term(null, \"query\")] cannot be null, received [null]", - error("from test | where term(null, \"query\")") - ); - assertEquals( - "1:19: second argument of [term(first_name, null)] cannot be null, received [null]", - error("from test | where term(first_name, null)") - ); + public void testFullTextFunctionsTargetsExistingField() throws Exception { + testFullTextFunctionTargetsExistingField("match(first_name, \"Anna\")"); + testFullTextFunctionTargetsExistingField("first_name : \"Anna\""); + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + testFullTextFunctionTargetsExistingField("multi_match(\"Anna\", first_name)"); + } + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + testFullTextFunctionTargetsExistingField("term(fist_name, \"Anna\")"); + } + if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { + testFullTextFunctionTargetsExistingField("knn(vector, [0, 1, 2])"); + } } - public void testTermTargetsExistingField() throws Exception { - assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); - assertEquals("1:38: Unknown column [first_name]", error("from test | keep emp_no | where term(first_name, \"Anna\")")); + private void testFullTextFunctionTargetsExistingField(String functionInvocation) throws Exception { + assertThat(error("from test | keep emp_no | where " + functionInvocation), containsString("Unknown column")); } public void testConditionalFunctionsWithMixedNumericTypes() { @@ -2090,7 +2034,7 @@ public void testLookupJoinDataTypeMismatch() { public void testFullTextFunctionOptions() { checkOptionDataTypes(Match.ALLOWED_OPTIONS, "FROM test | WHERE match(first_name, \"Jean\", {\"%s\": %s})"); checkOptionDataTypes(QueryString.ALLOWED_OPTIONS, "FROM test | WHERE QSTR(\"first_name: Jean\", {\"%s\": %s})"); - if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()){ + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"%s\": %s})"); } if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { @@ -2158,7 +2102,7 @@ public void testMultiMatchFunctionArgNotConstant() throws Exception { } // Should pass eventually once we lift some restrictions on full text search functions. - public void testMultiMatchFunctionCurrentlyUnsupportedBehaviour() throws Exception { + public void testFullTextFunctionCurrentlyUnsupportedBehaviour() throws Exception { testFullTextFunctionsCurrentlyUnsupportedBehaviour("match(first_name, \"Anna\")"); testFullTextFunctionsCurrentlyUnsupportedBehaviour("first_name : \"Anna\""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { @@ -2176,30 +2120,51 @@ private void testFullTextFunctionsCurrentlyUnsupportedBehaviour(String functionI ); } - public void testMultiMatchFunctionNullArgs() throws Exception { - assertEquals( - "1:19: first argument of [multi_match(\"query\", null)] cannot be null, received [null]", - error("from test | where multi_match(\"query\", null)") - ); - assertEquals( - "1:19: first argument of [multi_match(null, first_name)] cannot be null, received [null]", - error("from test | where multi_match(null, first_name)") - ); + public void testFullTextFunctionsNullArgs() throws Exception { + testFullTextFunctionNullArgs("match(null, \"query\")", "first"); + testFullTextFunctionNullArgs("match(first_name, null)", "second"); + testFullTextFunctionNullArgs("qstr(null)", ""); + testFullTextFunctionNullArgs("kql(null)", ""); + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + testFullTextFunctionNullArgs("multi_match(null, first_name)", "first"); + testFullTextFunctionNullArgs("multi_match(\"query\", null)", "second"); + } + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + testFullTextFunctionNullArgs("term(null, \"query\")", "first"); + testFullTextFunctionNullArgs("term(first_name, null)", "second"); + } + if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { + testFullTextFunctionNullArgs("knn(null, [0, 1, 2])", "first"); + testFullTextFunctionNullArgs("knn(vector, null)", "second"); + } } - public void testMultiMatchTargetsExistingField() throws Exception { - assertEquals( - "1:53: Unknown column [first_name]\nline 1:65: Unknown column [last_name]", - error("from test | keep emp_no | where multi_match(\"Anna\", first_name, last_name)") + private void testFullTextFunctionNullArgs(String functionInvocation, String argOrdinal) throws Exception { + assertThat( + error("from test | where " + functionInvocation), + containsString(argOrdinal + " argument of [" + functionInvocation + "] cannot be null, received [null]") ); } - public void testMultiMatchInsideEval() throws Exception { - assumeTrue("MultiMatch operator is available just for snapshots", Build.current().isSnapshot()); - assertEquals( - "1:36: [MultiMatch] function is only supported in WHERE and STATS commands\n" - + "line 1:55: [MultiMatch] function cannot operate on [title], which is not a field from an index mapping", - error("row title = \"brown fox\" | eval x = multi_match(\"fox\", title)") + public void testFullTextFunctionsConstantQuery() throws Exception { + testFullTextFunctionsConstantQuery("match(first_name, last_name)", "second"); + testFullTextFunctionsConstantQuery("qstr(first_name)", ""); + testFullTextFunctionsConstantQuery("kql(first_name)", ""); + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + testFullTextFunctionsConstantQuery("multi_match(first_name, first_name)", "first"); + } + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + testFullTextFunctionsConstantQuery("term(first_name, last_name)", "second"); + } + if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { + testFullTextFunctionsConstantQuery("knn(vector, vector)", "second"); + } + } + + private void testFullTextFunctionsConstantQuery(String functionInvocation, String argOrdinal) throws Exception { + assertThat( + error("from test | where " + functionInvocation), + containsString(argOrdinal + " argument of [" + functionInvocation + "] must be a constant") ); } From 6580832952c1fe6d4a621ba88e5f426fd9fd869b Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Mon, 2 Jun 2025 18:02:23 +0200 Subject: [PATCH 06/14] Fix merge --- .../xpack/esql/analysis/VerifierTests.java | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index e7001d4134b79..154fe3e1c2ea4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -54,7 +54,6 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; -import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.hamcrest.Matchers.containsString; @@ -1223,7 +1222,6 @@ public void testFieldBasedFullTextFunctions() throws Exception { testFieldBasedFunctionNotAllowedAfterCommands("MATCH", "function", "match(first_name, \"Anna\")"); testFieldBasedFunctionNotAllowedAfterCommands(":", "operator", "first_name : \"Anna\""); testFieldBasedFunctionNotAllowedAfterCommands("MultiMatch", "function", "multi_match(\"Anna\", first_name)"); - testFieldBasedFunctionNotAllowedAfterCommands("KNN", "function", "knn(vector, [1, 2, 3])"); } public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, String functionType, String functionInvocation) @@ -1351,9 +1349,6 @@ public void testFullTextFunctionsOnlyAllowedInWhere() throws Exception { if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkFullTextFunctionsOnlyAllowedInWhere("MultiMatch", "multi_match(\"Anna\", first_name, last_name)", "function"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { - checkFullTextFunctionsOnlyAllowedInWhere("KNN", "knn(vector, [1, 2, 3])", "function"); - } } private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation, String functionType) @@ -1437,9 +1432,6 @@ public void testFullTextFunctionsWithNonBooleanFunctions() { if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkFullTextFunctionsWithNonBooleanFunctions("MultiMatch", "multi_match(\"Anna\", first_name, last_name)", "function"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { - checkFullTextFunctionsWithNonBooleanFunctions("KNN", "knn(vector, [1, 2, 3])", "function"); - } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { checkFullTextFunctionsWithNonBooleanFunctions("Term", "term(first_name, \"Anna\")", "function"); } @@ -1510,9 +1502,6 @@ public void testFullTextFunctionsTargetsExistingField() throws Exception { if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { testFullTextFunctionTargetsExistingField("term(fist_name, \"Anna\")"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { - testFullTextFunctionTargetsExistingField("knn(vector, [0, 1, 2])"); - } } private void testFullTextFunctionTargetsExistingField(String functionInvocation) throws Exception { @@ -2037,9 +2026,6 @@ public void testFullTextFunctionOptions() { if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"%s\": %s})"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { - checkOptionDataTypes(Knn.ALLOWED_OPTIONS, "FROM test | WHERE KNN(vector, [0.1, 0.2, 0.3], {\"%s\": %s})"); - } } /** @@ -2133,10 +2119,6 @@ public void testFullTextFunctionsNullArgs() throws Exception { testFullTextFunctionNullArgs("term(null, \"query\")", "first"); testFullTextFunctionNullArgs("term(first_name, null)", "second"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { - testFullTextFunctionNullArgs("knn(null, [0, 1, 2])", "first"); - testFullTextFunctionNullArgs("knn(vector, null)", "second"); - } } private void testFullTextFunctionNullArgs(String functionInvocation, String argOrdinal) throws Exception { @@ -2156,9 +2138,6 @@ public void testFullTextFunctionsConstantQuery() throws Exception { if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { testFullTextFunctionsConstantQuery("term(first_name, last_name)", "second"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION.isEnabled()) { - testFullTextFunctionsConstantQuery("knn(vector, vector)", "second"); - } } private void testFullTextFunctionsConstantQuery(String functionInvocation, String argOrdinal) throws Exception { From 43997d86cbb06524e5f0dd91fe70bc0c81f17e39 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Tue, 3 Jun 2025 08:57:24 +0200 Subject: [PATCH 07/14] Add capabilities checks --- .../xpack/esql/analysis/VerifierTests.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 154fe3e1c2ea4..21d133c01b282 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1216,12 +1216,19 @@ public void testMatchInsideEval() throws Exception { public void testFieldBasedFullTextFunctions() throws Exception { testFieldBasedWithNonIndexedColumn("MATCH", "match(text, \"cat\")", "function"); - testFieldBasedWithNonIndexedColumn(":", "text : \"cat\"", "operator"); - testFieldBasedWithNonIndexedColumn("MultiMatch", "multi_match(\"cat\", text)", "function"); - testFieldBasedFunctionNotAllowedAfterCommands("MATCH", "function", "match(first_name, \"Anna\")"); + + testFieldBasedWithNonIndexedColumn(":", "text : \"cat\"", "operator"); testFieldBasedFunctionNotAllowedAfterCommands(":", "operator", "first_name : \"Anna\""); - testFieldBasedFunctionNotAllowedAfterCommands("MultiMatch", "function", "multi_match(\"Anna\", first_name)"); + + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + testFieldBasedWithNonIndexedColumn("MultiMatch", "multi_match(\"cat\", text)", "function"); + testFieldBasedFunctionNotAllowedAfterCommands("MultiMatch", "function", "multi_match(\"Anna\", first_name)"); + } + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + testFieldBasedWithNonIndexedColumn("Term", "term(text, \"cat\")", "function"); + testFieldBasedFunctionNotAllowedAfterCommands("Term", "function", "term(first_name, \"Anna\")"); + } } public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, String functionType, String functionInvocation) @@ -1375,10 +1382,12 @@ private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, Strin public void testFullTextFunctionsDisjunctions() { checkWithFullTextFunctionsDisjunctions("match(last_name, \"Smith\")"); - checkWithFullTextFunctionsDisjunctions("multi_match(\"Smith\", first_name, last_name)"); checkWithFullTextFunctionsDisjunctions("last_name : \"Smith\""); checkWithFullTextFunctionsDisjunctions("qstr(\"last_name: Smith\")"); checkWithFullTextFunctionsDisjunctions("kql(\"last_name: Smith\")"); + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + checkWithFullTextFunctionsDisjunctions("multi_match(\"Smith\", first_name, last_name)"); + } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { checkWithFullTextFunctionsDisjunctions("term(last_name, \"Smith\")"); } @@ -2076,17 +2085,6 @@ private static String exampleValueForType(DataType currentType) { }; } - public void testMultiMatchFunctionArgNotConstant() throws Exception { - assertEquals( - "1:19: second argument of [match(first_name, first_name)] must be a constant, received [first_name]", - error("from test | where match(first_name, first_name)") - ); - assertEquals( - "1:59: second argument of [match(first_name, query)] must be a constant, received [query]", - error("from test | eval query = concat(\"first\", \" name\") | where match(first_name, query)") - ); - } - // Should pass eventually once we lift some restrictions on full text search functions. public void testFullTextFunctionCurrentlyUnsupportedBehaviour() throws Exception { testFullTextFunctionsCurrentlyUnsupportedBehaviour("match(first_name, \"Anna\")"); @@ -2102,7 +2100,7 @@ public void testFullTextFunctionCurrentlyUnsupportedBehaviour() throws Exception private void testFullTextFunctionsCurrentlyUnsupportedBehaviour(String functionInvocation) throws Exception { assertThat( error("from test | stats max_salary = max(salary) by emp_no | where " + functionInvocation), - containsString("Unknown column [first_name]") + containsString("Unknown column") ); } @@ -2134,6 +2132,7 @@ public void testFullTextFunctionsConstantQuery() throws Exception { testFullTextFunctionsConstantQuery("kql(first_name)", ""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { testFullTextFunctionsConstantQuery("multi_match(first_name, first_name)", "first"); + testFullTextFunctionsConstantQuery("multi_match(concat(first_name, \"world\"), first_name)", "first"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { testFullTextFunctionsConstantQuery("term(first_name, last_name)", "second"); @@ -2158,14 +2157,15 @@ public void testInsistNotOnTopOfFrom() { public void testFullTextFunctionsInStats() { checkFullTextFunctionsInStats("match(last_name, \"Smith\")"); - checkFullTextFunctionsInStats("multi_match(\"Smith\", first_name, last_name)"); checkFullTextFunctionsInStats("last_name : \"Smith\""); checkFullTextFunctionsInStats("qstr(\"last_name: Smith\")"); checkFullTextFunctionsInStats("kql(\"last_name: Smith\")"); + if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { + checkFullTextFunctionsInStats("multi_match(\"Smith\", first_name, last_name)"); + } } private void checkFullTextFunctionsInStats(String functionInvocation) { - query("from test | stats c = max(salary) where " + functionInvocation); query("from test | stats c = max(salary) where " + functionInvocation + " or length(first_name) > 10"); query("from test metadata _score | where " + functionInvocation + " | stats c = max(_score)"); From 713de2bd2fc42098210bae9a29f568b2a402b50d Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Tue, 3 Jun 2025 11:38:02 +0200 Subject: [PATCH 08/14] Add new full text functions data set and modify VerifierTests --- .../xpack/esql/CsvTestsDataLoader.java | 6 +- .../main/resources/data/full_text_search.csv | 21 ++ .../src/main/resources/knn-function.csv-spec | 321 ++++++++++++++++++ .../resources/mapping-full_text_search.json | 26 ++ .../xpack/esql/analysis/VerifierTests.java | 306 +++++++++-------- 5 files changed, 529 insertions(+), 151 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/full_text_search.csv create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-full_text_search.json diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 92fe597362bb0..66fca56efc1a8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -144,6 +144,8 @@ public class CsvTestsDataLoader { private static final TestDataset LOGS = new TestDataset("logs"); private static final TestDataset MV_TEXT = new TestDataset("mv_text"); private static final TestDataset DENSE_VECTOR = new TestDataset("dense_vector"); + private static final TestDataset COLORS = new TestDataset("colors"); + private static final TestDataset FULL_TEXT_SEARCH = new TestDataset("full_text_search"); public static final Map CSV_DATASET_MAP = Map.ofEntries( Map.entry(EMPLOYEES.indexName, EMPLOYEES), @@ -204,7 +206,9 @@ public class CsvTestsDataLoader { Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT), Map.entry(LOGS.indexName, LOGS), Map.entry(MV_TEXT.indexName, MV_TEXT), - Map.entry(DENSE_VECTOR.indexName, DENSE_VECTOR) + Map.entry(DENSE_VECTOR.indexName, DENSE_VECTOR), + Map.entry(COLORS.indexName, COLORS), + Map.entry(FULL_TEXT_SEARCH.indexName, FULL_TEXT_SEARCH) ); private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json"); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/full_text_search.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/full_text_search.csv new file mode 100644 index 0000000000000..8d0fecdb4f8aa --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/full_text_search.csv @@ -0,0 +1,21 @@ +id:integer,title:text,body:text,tags:keyword,category:keyword,published_date:date,vector:dense_vector +1,The Rise of AI,Artificial intelligence is revolutionizing industries, from healthcare to finance.,ai,technology,future,technology,2023-01-15,[0.89, 0.61, 0.13] +2,Hiking the Grand Canyon,Exploring the vast landscapes of the Grand Canyon is an unforgettable experience.,travel,nature,hiking,travel,2022-11-20,[0.31, 0.85, 0.44] +3,Understanding Quantum Computing,Quantum computing leverages the principles of quantum mechanics for computation.,quantum,computing,research,science,2023-06-05,[0.92, 0.47, 0.22] +4,Healthy Meal Planning,Meal prepping with nutritious ingredients can save time and improve well-being.,health,food,planning,lifestyle,2024-03-08,[0.41, 0.66, 0.30] +5,Dogs: Loyal Companions,Dogs provide emotional support and are known for their loyalty and affection.,dogs,pets,companionship,animals,2021-12-30,[0.14, 0.91, 0.19] +6,A Guide to the Solar System,The solar system is home to eight planets, each with unique characteristics.,space,planets,astronomy,science,2022-07-14,[0.75, 0.34, 0.56] +7,Meditation for Beginners,Meditation can help reduce stress and improve mental clarity when practiced regularly.,meditation,wellness,mental health,lifestyle,2023-02-28,[0.36, 0.72, 0.28] +8,Exploring Tokyo,Tokyo blends modern skyscrapers with traditional temples and vibrant street culture.,japan,tokyo,travel,travel,2024-09-10,[0.45, 0.82, 0.39] +9,Introduction to Neural Networks,Neural networks are foundational to deep learning, a subfield of machine learning.,neural networks,deep learning,ai,technology,2023-10-01,[0.88, 0.60, 0.15] +10,Gardening Tips for Spring,Spring is the ideal time to start planting flowers, herbs, and vegetables.,gardening,plants,spring,lifestyle,2022-03-15,[0.33, 0.76, 0.21] +11,Basics of Blockchain Technology,Blockchain provides a decentralized way to store and verify transactions.,blockchain,cryptocurrency,tech,technology,2023-05-22,[0.91, 0.55, 0.20] +12,Cats vs Dogs,Cats and dogs are both popular pets, each with unique behaviors and needs.,cats,dogs,pets,animals,2022-08-18,[0.18, 0.89, 0.23] +13,The Benefits of Yoga,Yoga combines physical postures with breathing exercises and meditation.,yoga,fitness,health,lifestyle,2024-01-04,[0.40, 0.69, 0.27] +14,Visiting the Louvre Museum,The Louvre houses famous artworks like the Mona Lisa and Venus de Milo.,art,museum,paris,travel,2021-06-25,[0.50, 0.78, 0.41] +15,Climate Change Explained,Climate change is a global challenge affecting ecosystems and weather patterns.,climate,environment,science,science,2022-10-11,[0.66, 0.70, 0.45] +16,Intro to Programming with Python,Python is a versatile language for beginners and experts alike.,python,programming,code,technology,2023-11-30,[0.87, 0.64, 0.18] +17,Exploring the Amazon Rainforest,The Amazon Rainforest is rich in biodiversity and cultural heritage.,amazon,nature,exploration,travel,2023-07-19,[0.38, 0.84, 0.46] +18,The Basics of Nutrition,Balanced nutrition is essential for energy, growth, and health maintenance.,nutrition,food,health,lifestyle,2022-05-12,[0.43, 0.68, 0.29] +19,Mars Missions Update,NASA and private companies continue exploring Mars with rovers and future plans.,mars,space,exploration,science,2024-04-26,[0.79, 0.37, 0.52] +20,Bird Watching in Costa Rica,Costa Rica offers a paradise for bird watchers with hundreds of species.,birds,wildlife,nature,travel,2021-09-09,[0.29, 0.83, 0.38] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec new file mode 100644 index 0000000000000..de446123c4fcf --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec @@ -0,0 +1,321 @@ +knnSearch +required_capability: knn_function + +// tag::knn-function[] +from colors metadata _score +| where knn(rgb_vector, [0, 120, 0]) +| sort _score desc +// end::knn-function[] +| keep color, rgb_vector +; + +// tag::knn-function-result[] +color:text | rgb_vector:dense_vector +green | [0.0, 128.0, 0.0] +dark green | [0.0, 100.0, 0.0] +forest green | [34.0, 139.0, 34.0] +dark olive green | [85.0, 107.0, 47.0] +sea green | [46.0, 139.0, 87.0] +dark slate gray | [47.0, 79.0, 79.0] +olive drab | [107.0, 142.0, 35.0] +lime green | [50.0, 205.0, 50.0] +black | [0.0, 0.0, 0.0] +olive | [128.0, 128.0, 0.0] +// end::knn-function-result[] +; + +knnSearchWithKOption +required_capability: knn_function + +// tag::knn-function-options[] +from colors metadata _score +| where knn(rgb_vector, [0,255,255], {"k": 4}) +| sort _score desc +// end::knn-function-options[] +| keep color, rgb_vector +; + +color:text | rgb_vector:dense_vector +cyan | [0.0, 255.0, 255.0] +deep sky blue | [0.0, 191.0, 255.0] +dark turquoise | [0.0, 206.0, 209.0] +turquoise | [64.0, 224.0, 208.0] +; + +knnSearchWithSimilarityOption +required_capability: knn_function + +from colors metadata _score +| where knn(rgb_vector, [255,192,203], {"similarity": 40}) +| sort _score desc +| keep color, rgb_vector +; + +color:text | rgb_vector:dense_vector +pink | [255.0, 192.0, 203.0] +light pink | [255.0, 182.0, 193.0] +peach puff | [255.0, 218.0, 185.0] +bisque | [255.0, 228.0, 196.0] +thistle | [216.0, 191.0, 216.0] +wheat | [245.0, 222.0, 179.0] +; + +knnHybridSearch +required_capability: knn_function + +from colors metadata _score +| where match(color, "violet") or knn(rgb_vector, [238,130,238], {"boost": 10.0, "k": 5}) +| sort _score desc +| eval round_score = round(_score, 4) +| keep color, rgb_vector, round_score +; + +color:text | rgb_vector:dense_vector | round_score:double +violet | [238.0, 130.0, 238.0] | 13.9457 +blue violet | [138.0, 43.0, 226.0] | 3.0871 +dark violet | [148.0, 0.0, 211.0] | 3.0871 +medium violet red | [199.0, 21.0, 133.0] | 2.5355 +pale violet red | [219.0, 112.0, 147.0] | 2.5355 +orchid | [218.0, 112.0, 214.0] | 0.0083 +plum | [221.0, 160.0, 221.0] | 0.0071 +hot pink | [255.0, 105.0, 180.0] | 0.0024 +thistle | [216.0, 191.0, 216.0] | 0.0021 +; + +knnWithMultipleFunctions +required_capability: knn_function + +from colors metadata _score +| where knn(rgb_vector, [128,128,0]) and match(color, "olive") +| sort _score desc +| eval round_score = round(_score, 4) +| keep color, rgb_vector, round_score +; + +color:text | rgb_vector:dense_vector | round_score:double +olive | [128.0, 128.0, 0.0] | 5.4979 +olive drab | [107.0, 142.0, 35.0] | 3.5206 +dark olive green | [85.0, 107.0, 47.0] | 2.8906 +; + +knnAfterKeep +required_capability: knn_function + +from colors metadata _score +| keep rgb_vector, _score +| where knn(rgb_vector, [128,128,0]) +| eval round_score = round(_score, 4) +| sort round_score desc +| keep rgb_vector, round_score +| limit 5 +; + +rgb_vector:dense_vector | round_score:double +[128.0, 128.0, 0.0] | 1.0 +[107.0, 142.0, 35.0] | 0.0014 +[85.0, 107.0, 47.0] | 4.0E-4 +[139.0, 69.0, 19.0] | 3.0E-4 +[184.0, 134.0, 11.0] | 3.0E-4 +; + +knnAfterDrop +required_capability: knn_function + +from colors metadata _score +| drop color +| where knn(rgb_vector, [128,128,0]) +| eval round_score = round(_score, 4) +| keep rgb_vector, round_score +| limit 5 +; + +rgb_vector:dense_vector | round_score:double +[184.0, 134.0, 11.0] | 3.0E-4 +[128.0, 128.0, 0.0] | 1.0 +[154.0, 205.0, 50.0] | 1.0E-4 +[85.0, 107.0, 47.0] | 4.0E-4 +[107.0, 142.0, 35.0] | 0.0014 +; + +knnAfterEval +required_capability: knn_function + +from colors metadata _score +| eval composed_name = locate(color, " ") > 0 +| where knn(rgb_vector, [128,128,0]) +| sort _score, color desc +| keep color, composed_name +; + +color:text | composed_name:boolean +peru | false +yellow green | true +chocolate | false +dim gray | true +saddle brown | true +sienna | false +dark golden rod | true +dark olive green | true +olive drab | true +olive | false +; + +knnWithConjunction +required_capability: knn_function + +# TODO We need kNN prefiltering here so we get more candidates that pass the filter +from colors metadata _score +| where knn(rgb_vector, [255,255,238]) and hex_code like "#FFF*" +| keep color, hex_code, rgb_vector +; +ignoreOrder:true + +color:text | hex_code: keyword | rgb_vector:dense_vector +light yellow | #FFFFE0 | [255.0, 255.0, 224.0] +lavender blush | #FFF0F5 | [255.0, 240.0, 245.0] +sea shell | #FFF5EE | [255.0, 245.0, 238.0] +floral white | #FFFAF0 | [255.0, 250.0, 240.0] +ivory | #FFFFF0 | [255.0, 255.0, 240.0] +snow | #FFFAFA | [255.0, 250.0, 250.0] +white | #FFFFFF | [255.0, 255.0, 255.0] +; + +knnWithDisjunctionAndFiltersConjunction +required_capability: knn_function + +# TODO We need kNN prefiltering here so we get more candidates that pass the filter +from colors metadata _score +| where (knn(rgb_vector, [0,255,255]) or knn(rgb_vector, [128, 0, 255])) and primary == true +| keep color, rgb_vector, _score +; + +color:text | rgb_vector:dense_vector | _score:double +cyan | [0.0, 255.0, 255.0] | 1.0 +blue | [0.0, 0.0, 255.0] | 9.922293975250795E-5 +; + +knnWithDisjunctionAndConjunction +required_capability: knn_function +required_capability: full_text_functions_disjunctions + +# TODO We need kNN prefiltering here so we get more candidates that pass the filter +from colors metadata _score +| where (knn(rgb_vector, [0,255,255]) or knn(rgb_vector, [0, 0, 255])) and knn(rgb_vector, [0, 255, 0]) +| keep color, rgb_vector, _score +; + +color:text | rgb_vector:dense_vector | _score:double +medium spring green | [0.0, 250.0, 154.0] | 1.6871128173079342E-4 +; + +knnWithNonPushableConjunction +required_capability: knn_function + +from colors metadata _score +| eval composed_name = locate(color, " ") > 0 +| where knn(rgb_vector, [128,128,0]) and composed_name == false +| eval round_score = round(_score, 4) +| keep color, composed_name, round_score +; + +color:text | composed_name:boolean | round_score:double +olive | false | 1.0 +sienna | false | 3.0E-4 +chocolate | false | 1.0E-4 +peru | false | 1.0E-4 +; + +testKnnWithNonPushableDisjunctions +required_capability: knn_function + +from colors metadata _score +| where knn(rgb_vector, [128,128,0], {"k": 5}) or length(color) > 17 +| sort _score desc, color asc +| eval round_score = round(_score, 4) +| keep color, round_score +; + +color:text | round_score: double +olive | 1.0 +olive drab | 0.0014 +dark olive green | 4.0E-4 +dark golden rod | 3.0E-4 +sienna | 3.0E-4 +medium aqua marine | 0.0 +medium spring green | 0.0 +light golden rod yellow | 0.0 +; + +testKnnWithNonPushableDisjunctionsOnComplexExpressions +required_capability: knn_function + +from colors metadata _score +| where (knn(rgb_vector, [128,128,0]) and length(color) > 12) or (knn(rgb_vector, [128,0,128]) and primary == false) +| sort _score desc +| eval round_score = round(_score, 4) +| keep color, primary, round_score +; + +color: text | primary: boolean | round_score: double +purple | false | 1.0 +dark magenta | false | 0.0045 +dark olive green | false | 4.0E-4 +indigo | false | 4.0E-4 +dark golden rod | false | 3.0E-4 +dim gray | false | 3.0E-4 +dark slate blue | false | 2.0E-4 +medium violet red | false | 2.0E-4 +dark orchid | false | 1.0E-4 +dark violet | false | 1.0E-4 +brown | false | 1.0E-4 +blue violet | false | 1.0E-4 +; + +testKnnInStatsNonPushable +required_capability: knn_function + +from colors +| where length(color) < 10 +| stats c = count(*) where knn(rgb_vector, [128,128,255], {"k": 140}) +; + +c: long +59 +; + + +testKnnInStatsPushableAndNonPushable +required_capability: knn_function +required_capability: full_text_functions_in_stats_where + +from colors metadata _score +| stats c = count(*) where (knn(rgb_vector, [0,255,255], {"k": 140}) or knn(rgb_vector, [0, 0, 255])) and knn(rgb_vector, [0, 255, 0], {"k": 40}) +; + +c:long +40 +; + +testKnnInStatsWithGrouping +from colors +| where length(color) < 10 +| stats c = count(*) where knn(rgb_vector, [128,128,255], {"k": 140}) by primary +; + +c: long | primary: boolean +50 | false +9 | true +; + +testKnnInStatsPushable +required_capability: knn_function +required_capability: full_text_functions_in_stats_where + +from colors +| stats c = count(*) where knn(rgb_vector, [128,128,255], {"k": 40}) +; + +# No surprises, gets the number of top k +c:long +40 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-full_text_search.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-full_text_search.json new file mode 100644 index 0000000000000..160f285d792d1 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-full_text_search.json @@ -0,0 +1,26 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "body": { + "type": "text" + }, + "tags": { + "type": "keyword" + }, + "category": { + "type": "integer" + }, + "published_date": { + "type": "date" + }, + "vector": { + "type": "dense_vector", + "similarity": "l2_norm" + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 21d133c01b282..4e9c2fddba32e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -66,6 +66,7 @@ public class VerifierTests extends ESTestCase { private static final EsqlParser parser = new EsqlParser(); private final Analyzer defaultAnalyzer = AnalyzerTestUtils.expandedDefaultAnalyzer(); + private final Analyzer fullTextAnalyzer = AnalyzerTestUtils.analyzer(loadMapping("mapping-full_text_search.json", "test")); private final Analyzer tsdb = AnalyzerTestUtils.analyzer(AnalyzerTestUtils.tsdbIndexResolution()); private final List TIME_DURATIONS = List.of("millisecond", "second", "minute", "hour"); @@ -1216,30 +1217,30 @@ public void testMatchInsideEval() throws Exception { public void testFieldBasedFullTextFunctions() throws Exception { testFieldBasedWithNonIndexedColumn("MATCH", "match(text, \"cat\")", "function"); - testFieldBasedFunctionNotAllowedAfterCommands("MATCH", "function", "match(first_name, \"Anna\")"); + testFieldBasedFunctionNotAllowedAfterCommands("MATCH", "function", "match(title, \"Meditation\")"); testFieldBasedWithNonIndexedColumn(":", "text : \"cat\"", "operator"); - testFieldBasedFunctionNotAllowedAfterCommands(":", "operator", "first_name : \"Anna\""); + testFieldBasedFunctionNotAllowedAfterCommands(":", "operator", "title : \"Meditation\""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { testFieldBasedWithNonIndexedColumn("MultiMatch", "multi_match(\"cat\", text)", "function"); - testFieldBasedFunctionNotAllowedAfterCommands("MultiMatch", "function", "multi_match(\"Anna\", first_name)"); + testFieldBasedFunctionNotAllowedAfterCommands("MultiMatch", "function", "multi_match(\"Meditation\", title)"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { testFieldBasedWithNonIndexedColumn("Term", "term(text, \"cat\")", "function"); - testFieldBasedFunctionNotAllowedAfterCommands("Term", "function", "term(first_name, \"Anna\")"); + testFieldBasedFunctionNotAllowedAfterCommands("Term", "function", "term(title, \"Meditation\")"); } } public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, String functionType, String functionInvocation) throws Exception { assertThat( - error("from test | limit 10 | where " + functionInvocation), + error("from test | limit 10 | where " + functionInvocation, fullTextAnalyzer), containsString("[" + functionName + "] " + functionType + " cannot be used after LIMIT") ); - String fieldName = "KNN".equals(functionName) ? "vector" : "first_name"; + String fieldName = "KNN".equals(functionName) ? "vector" : "title"; assertThat( - error("from test | STATS c = COUNT(emp_no) BY " + fieldName + " | where " + functionInvocation), + error("from test | STATS c = COUNT(id) BY " + fieldName + " | where " + functionInvocation, fullTextAnalyzer), containsString("[" + functionName + "] " + functionType + " cannot be used after STATS") ); } @@ -1247,202 +1248,204 @@ public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, S // These should pass eventually once we lift some restrictions on match function public void testFieldBasedWithNonIndexedColumn(String functionName, String functionInvocation, String functionType) { assertThat( - error("from test | eval text = substring(first_name, 1) | where " + functionInvocation), + error("from test | eval text = substring(title, 1) | where " + functionInvocation, fullTextAnalyzer), containsString( "[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping" ) ); assertThat( - error("from test | eval text=concat(first_name, last_name) | where " + functionInvocation), + error("from test | eval text=concat(title, body) | where " + functionInvocation, fullTextAnalyzer), containsString( "[" + functionName + "] " + functionType + " cannot operate on [text], which is not a field from an index mapping" ) ); var keywordInvocation = functionInvocation.replace("text", "text::keyword"); - String keywordError = error("row n = null | eval text = n + 5 | where " + keywordInvocation); + String keywordError = error("row n = null | eval text = n + 5 | where " + keywordInvocation, fullTextAnalyzer); assertThat(keywordError, containsString("[" + functionName + "] " + functionType + " cannot operate on")); assertThat(keywordError, containsString("which is not a field from an index mapping")); } public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { - testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("QSTR", "qstr(\"field_name: Anna\")"); + testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("QSTR", "qstr(\"field_name: Meditation\")"); } public void testKqlFunctionsNotAllowedAfterCommands() throws Exception { - testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("KQL", "kql(\"field_name: Anna\")"); + testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("KQL", "kql(\"field_name: Meditation\")"); } public void testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands(String functionName, String functionInvocation) throws Exception { // Source commands - assertEquals("1:13: [" + functionName + "] function cannot be used after SHOW", error("show info | where " + functionInvocation)); - assertEquals( - "1:17: [" + functionName + "] function cannot be used after ROW", - error("row a= \"Anna\" | where " + functionInvocation) + assertThat( + error("show info | where " + functionInvocation), + containsString("[" + functionName + "] function cannot be used after SHOW") + ); + assertThat( + error("row a= \"Meditation\" | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after ROW") ); // Processing commands - assertEquals( - "1:43: [" + functionName + "] function cannot be used after DISSECT", - error("from test | dissect first_name \"%{foo}\" | where " + functionInvocation) + assertThat( + error("from test | dissect title \"%{foo}\" | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after DISSECT") ); - assertEquals( - "1:27: [" + functionName + "] function cannot be used after DROP", - error("from test | drop emp_no | where " + functionInvocation) + assertThat( + error("from test | drop body | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after DROP") ); - assertEquals( - "1:71: [" + functionName + "] function cannot be used after ENRICH", - error("from test | enrich languages on languages with lang = language_name | where " + functionInvocation) + assertThat( + error("from test | enrich languages on category with lang = language_name | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after ENRICH") ); - assertEquals( - "1:26: [" + functionName + "] function cannot be used after EVAL", - error("from test | eval z = 2 | where " + functionInvocation) + assertThat( + error("from test | eval z = 2 | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after EVAL") ); - assertEquals( - "1:44: [" + functionName + "] function cannot be used after GROK", - error("from test | grok last_name \"%{WORD:foo}\" | where " + functionInvocation) + assertThat( + error("from test | grok body \"%{WORD:foo}\" | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after GROK") ); - assertEquals( - "1:27: [" + functionName + "] function cannot be used after KEEP", - error("from test | keep emp_no | where " + functionInvocation) + assertThat( + error("from test | keep category | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after KEEP") ); - assertEquals( - "1:24: [" + functionName + "] function cannot be used after LIMIT", - error("from test | limit 10 | where " + functionInvocation) + assertThat( + error("from test | limit 10 | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after LIMIT") ); - assertEquals( - "1:35: [" + functionName + "] function cannot be used after MV_EXPAND", - error("from test | mv_expand last_name | where " + functionInvocation) + assertThat( + error("from test | mv_expand body | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after MV_EXPAND") ); - assertEquals( - "1:45: [" + functionName + "] function cannot be used after RENAME", - error("from test | rename last_name as full_name | where " + functionInvocation) + assertThat( + error("from test | rename body as full_body | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after RENAME") ); - assertEquals( - "1:52: [" + functionName + "] function cannot be used after STATS", - error("from test | STATS c = COUNT(emp_no) BY languages | where " + functionInvocation) + assertThat( + error("from test | STATS c = COUNT(*) BY category | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after STATS") ); // Some combination of processing commands - assertEquals( - "1:38: [" + functionName + "] function cannot be used after LIMIT", - error("from test | keep emp_no | limit 10 | where " + functionInvocation) + assertThat( + error("from test | keep category | limit 10 | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after LIMIT") ); - assertEquals( - "1:46: [" + functionName + "] function cannot be used after MV_EXPAND", - error("from test | limit 10 | mv_expand last_name | where " + functionInvocation) + assertThat( + error("from test | limit 10 | mv_expand body | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after MV_EXPAND") ); - assertEquals( - "1:52: [" + functionName + "] function cannot be used after KEEP", - error("from test | mv_expand last_name | keep last_name | where " + functionInvocation) + assertThat( + error("from test | mv_expand body | keep body | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after KEEP") ); - assertEquals( - "1:77: [" + functionName + "] function cannot be used after RENAME", - error("from test | STATS c = COUNT(emp_no) BY languages | rename c as total_emps | where " + functionInvocation) + assertThat( + error("from test | STATS c = COUNT(id) BY category | rename c as total_categories | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after RENAME") ); - assertEquals( - "1:54: [" + functionName + "] function cannot be used after DROP", - error("from test | rename last_name as name | drop emp_no | where " + functionInvocation) + assertThat( + error("from test | rename title as name | drop category | where " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] function cannot be used after DROP") ); } public void testFullTextFunctionsOnlyAllowedInWhere() throws Exception { - checkFullTextFunctionsOnlyAllowedInWhere("MATCH", "match(first_name, \"Anna\")", "function"); - checkFullTextFunctionsOnlyAllowedInWhere(":", "first_name:\"Anna\"", "operator"); - checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")", "function"); - checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function"); + checkFullTextFunctionsOnlyAllowedInWhere("MATCH", "match(title, \"Meditation\")", "function"); + checkFullTextFunctionsOnlyAllowedInWhere(":", "title:\"Meditation\"", "operator"); + checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Meditation\")", "function"); + checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Meditation\")", "function"); if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - checkFullTextFunctionsOnlyAllowedInWhere("Term", "term(first_name, \"Anna\")", "function"); + checkFullTextFunctionsOnlyAllowedInWhere("Term", "term(title, \"Meditation\")", "function"); } if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - checkFullTextFunctionsOnlyAllowedInWhere("MultiMatch", "multi_match(\"Anna\", first_name, last_name)", "function"); + checkFullTextFunctionsOnlyAllowedInWhere("MultiMatch", "multi_match(\"Meditation\", title, body)", "function"); } } private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation, String functionType) throws Exception { - assertEquals( - "1:22: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", - error("from test | eval y = " + functionInvocation) + assertThat( + error("from test | eval y = " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands") ); - assertEquals( - "1:18: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", - error("from test | sort " + functionInvocation + " asc") + assertThat( + error("from test | sort " + functionInvocation + " asc", fullTextAnalyzer), + containsString("[" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands") ); - assertEquals( - "1:47: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", - error("from test | stats max_salary = max(salary) by " + functionInvocation) + assertThat( + error("from test | stats max_id = max(id) by " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands") ); if ("KQL".equals(functionName) || "QSTR".equals(functionName)) { - assertEquals( - "1:9: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", - error("row a = " + functionInvocation) + assertThat( + error("row a = " + functionInvocation, fullTextAnalyzer), + containsString("[" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands") ); } } public void testFullTextFunctionsDisjunctions() { - checkWithFullTextFunctionsDisjunctions("match(last_name, \"Smith\")"); - checkWithFullTextFunctionsDisjunctions("last_name : \"Smith\""); - checkWithFullTextFunctionsDisjunctions("qstr(\"last_name: Smith\")"); - checkWithFullTextFunctionsDisjunctions("kql(\"last_name: Smith\")"); + checkWithFullTextFunctionsDisjunctions("match(title, \"Meditation\")"); + checkWithFullTextFunctionsDisjunctions("title : \"Meditation\""); + checkWithFullTextFunctionsDisjunctions("qstr(\"title: Meditation\")"); + checkWithFullTextFunctionsDisjunctions("kql(\"title: Meditation\")"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - checkWithFullTextFunctionsDisjunctions("multi_match(\"Smith\", first_name, last_name)"); + checkWithFullTextFunctionsDisjunctions("multi_match(\"Meditation\", title, body)"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - checkWithFullTextFunctionsDisjunctions("term(last_name, \"Smith\")"); + checkWithFullTextFunctionsDisjunctions("term(title, \"Meditation\")"); } } private void checkWithFullTextFunctionsDisjunctions(String functionInvocation) { // Disjunctions with non-pushable functions - scoring - query("from test | where " + functionInvocation + " or length(first_name) > 10"); - query("from test | where match(last_name, \"Anneke\") or (" + functionInvocation + " and length(first_name) > 10)"); + query("from test | where " + functionInvocation + " or length(title) > 10", fullTextAnalyzer); + query("from test | where match(title, \"Meditation\") or (" + functionInvocation + " and length(title) > 10)", fullTextAnalyzer); query( "from test | where (" + functionInvocation - + " and length(first_name) > 0) or (match(last_name, \"Anneke\") and length(first_name) > 10)" + + " and length(title) > 0) or (match(title, \"Meditation\") and length(title) > 10)", fullTextAnalyzer ); // Disjunctions with non-pushable functions - no scoring - query("from test | where " + functionInvocation + " or length(first_name) > 10"); - query("from test | where match(last_name, \"Anneke\") or (" + functionInvocation + " and length(first_name) > 10)"); + query("from test | where " + functionInvocation + " or length(title) > 10", fullTextAnalyzer); + query("from test | where match(title, \"Meditation\") or (" + functionInvocation + " and length(title) > 10)", fullTextAnalyzer); query( "from test | where (" + functionInvocation - + " and length(first_name) > 0) or (match(last_name, \"Anneke\") and length(first_name) > 10)" + + " and length(title) > 0) or (match(title, \"Meditation\") and length(title) > 10)", fullTextAnalyzer ); // Disjunctions with full text functions - no scoring - query("from test | where " + functionInvocation + " or match(first_name, \"Anna\")"); - query("from test | where " + functionInvocation + " or not match(first_name, \"Anna\")"); - query("from test | where (" + functionInvocation + " or match(first_name, \"Anna\")) and length(first_name) > 10"); - query("from test | where (" + functionInvocation + " or match(first_name, \"Anna\")) and match(last_name, \"Smith\")"); - query("from test | where " + functionInvocation + " or (match(first_name, \"Anna\") and match(last_name, \"Smith\"))"); + query("from test | where " + functionInvocation + " or match(title, \"Meditation\")", fullTextAnalyzer); + query("from test | where " + functionInvocation + " or not match(title, \"Meditation\")", fullTextAnalyzer); + query("from test | where (" + functionInvocation + " or match(title, \"Meditation\")) and length(title) > 10", fullTextAnalyzer); + query("from test | where (" + functionInvocation + " or match(title, \"Meditation\")) and match(body, \"Smith\")", fullTextAnalyzer); + query("from test | where " + functionInvocation + " or (match(title, \"Meditation\") and match(body, \"Smith\"))", fullTextAnalyzer); // Disjunctions with full text functions - scoring - query("from test metadata _score | where " + functionInvocation + " or match(first_name, \"Anna\")"); - query("from test metadata _score | where " + functionInvocation + " or not match(first_name, \"Anna\")"); - query("from test metadata _score | where (" + functionInvocation + " or match(first_name, \"Anna\")) and length(first_name) > 10"); + query("from test metadata _score | where " + functionInvocation + " or match(title, \"Meditation\")", fullTextAnalyzer); + query("from test metadata _score | where " + functionInvocation + " or not match(title, \"Meditation\")", fullTextAnalyzer); + query("from test metadata _score | where (" + functionInvocation + " or match(title, \"Meditation\")) and length(title) > 10", fullTextAnalyzer); query( - "from test metadata _score | where (" + functionInvocation + " or match(first_name, \"Anna\")) and match(last_name, \"Smith\")" + "from test metadata _score | where (" + functionInvocation + " or match(title, \"Meditation\")) and match(body, \"Smith\")", fullTextAnalyzer ); query( - "from test metadata _score | where " + functionInvocation + " or (match(first_name, \"Anna\") and match(last_name, \"Smith\"))" + "from test metadata _score | where " + functionInvocation + " or (match(title, \"Meditation\") and match(body, \"Smith\"))", fullTextAnalyzer ); - } public void testFullTextFunctionsWithNonBooleanFunctions() { - checkFullTextFunctionsWithNonBooleanFunctions("MATCH", "match(first_name, \"Anna\")", "function"); - checkFullTextFunctionsWithNonBooleanFunctions(":", "first_name:\"Anna\"", "operator"); - checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); - checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function"); + checkFullTextFunctionsWithNonBooleanFunctions("MATCH", "match(title, \"Meditation\")", "function"); + checkFullTextFunctionsWithNonBooleanFunctions(":", "title:\"Meditation\"", "operator"); + checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"title: Meditation\")", "function"); + checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"title: Meditation\")", "function"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - checkFullTextFunctionsWithNonBooleanFunctions("MultiMatch", "multi_match(\"Anna\", first_name, last_name)", "function"); + checkFullTextFunctionsWithNonBooleanFunctions("MultiMatch", "multi_match(\"Meditation\", title, body)", "function"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - checkFullTextFunctionsWithNonBooleanFunctions("Term", "term(first_name, \"Anna\")", "function"); + checkFullTextFunctionsWithNonBooleanFunctions("Term", "term(title, \"Meditation\")", "function"); } } @@ -1457,7 +1460,7 @@ private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, + "] " + functionType + " can't be used with ISNOTNULL", - error("from test | where " + functionInvocation + " is not null") + error("from test | where " + functionInvocation + " is not null", fullTextAnalyzer) ); assertEquals( "1:19: Invalid condition [" @@ -1467,7 +1470,7 @@ private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, + "] " + functionType + " can't be used with ISNULL", - error("from test | where " + functionInvocation + " is null") + error("from test | where " + functionInvocation + " is null", fullTextAnalyzer) ); assertEquals( "1:19: Invalid condition [" @@ -1477,7 +1480,7 @@ private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, + "] " + functionType + " can't be used with IN", - error("from test | where " + functionInvocation + " in (\"hello\", \"world\")") + error("from test | where " + functionInvocation + " in (\"hello\", \"world\")", fullTextAnalyzer) ); } assertEquals( @@ -1490,7 +1493,7 @@ private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, + "] " + functionType + " can't be used with COALESCE", - error("from test | where coalesce(" + functionInvocation + ", " + functionInvocation + ")") + error("from test | where coalesce(" + functionInvocation + ", " + functionInvocation + ")", fullTextAnalyzer) ); assertEquals( "1:19: argument of [concat(" @@ -1498,23 +1501,26 @@ private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, + ", \"a\")] must be [string], found value [" + functionInvocation + "] type [boolean]", - error("from test | where concat(" + functionInvocation + ", \"a\")") + error("from test | where concat(" + functionInvocation + ", \"a\")", fullTextAnalyzer) ); } public void testFullTextFunctionsTargetsExistingField() throws Exception { - testFullTextFunctionTargetsExistingField("match(first_name, \"Anna\")"); - testFullTextFunctionTargetsExistingField("first_name : \"Anna\""); + testFullTextFunctionTargetsExistingField("match(title, \"Meditation\")"); + testFullTextFunctionTargetsExistingField("title : \"Meditation\""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - testFullTextFunctionTargetsExistingField("multi_match(\"Anna\", first_name)"); + testFullTextFunctionTargetsExistingField("multi_match(\"Meditation\", title)"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - testFullTextFunctionTargetsExistingField("term(fist_name, \"Anna\")"); + testFullTextFunctionTargetsExistingField("term(fist_name, \"Meditation\")"); } } private void testFullTextFunctionTargetsExistingField(String functionInvocation) throws Exception { - assertThat(error("from test | keep emp_no | where " + functionInvocation), containsString("Unknown column")); + assertThat( + error("from test | keep emp_no | where " + functionInvocation), + containsString("Unknown column") + ); } public void testConditionalFunctionsWithMixedNumericTypes() { @@ -2030,10 +2036,10 @@ public void testLookupJoinDataTypeMismatch() { } public void testFullTextFunctionOptions() { - checkOptionDataTypes(Match.ALLOWED_OPTIONS, "FROM test | WHERE match(first_name, \"Jean\", {\"%s\": %s})"); - checkOptionDataTypes(QueryString.ALLOWED_OPTIONS, "FROM test | WHERE QSTR(\"first_name: Jean\", {\"%s\": %s})"); + checkOptionDataTypes(Match.ALLOWED_OPTIONS, "FROM test | WHERE match(title, \"Jean\", {\"%s\": %s})"); + checkOptionDataTypes(QueryString.ALLOWED_OPTIONS, "FROM test | WHERE QSTR(\"title: Jean\", {\"%s\": %s})"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", first_name, last_name, {\"%s\": %s})"); + checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", title, body, {\"%s\": %s})"); } } @@ -2059,10 +2065,10 @@ private void checkOptionDataTypes(Map allowedOptionsMap, Strin // Check conversion is possible DataTypeConverter.convert(optionValue, optionType); // If no exception was thrown, conversion is possible and should be done - query(query); + query(query, fullTextAnalyzer); } catch (InvalidArgumentException e) { // Conversion is not possible, query should fail - String error = error(query); + String error = error(query, fullTextAnalyzer); assertThat(error, containsString("Invalid option [" + optionName + "]")); assertThat(error, containsString("cannot cast [" + optionValue + "] to [" + optionType.typeName() + "]")); } @@ -2070,7 +2076,7 @@ private void checkOptionDataTypes(Map allowedOptionsMap, Strin } String errorQuery = String.format(Locale.ROOT, queryTemplate, "unknown_option", "\"any_value\""); - assertThat(error(errorQuery), containsString("Invalid option [unknown_option]")); + assertThat(error(errorQuery, fullTextAnalyzer), containsString("Invalid option [unknown_option]")); } private static String exampleValueForType(DataType currentType) { @@ -2087,61 +2093,61 @@ private static String exampleValueForType(DataType currentType) { // Should pass eventually once we lift some restrictions on full text search functions. public void testFullTextFunctionCurrentlyUnsupportedBehaviour() throws Exception { - testFullTextFunctionsCurrentlyUnsupportedBehaviour("match(first_name, \"Anna\")"); - testFullTextFunctionsCurrentlyUnsupportedBehaviour("first_name : \"Anna\""); + testFullTextFunctionsCurrentlyUnsupportedBehaviour("match(title, \"Meditation\")"); + testFullTextFunctionsCurrentlyUnsupportedBehaviour("title : \"Meditation\""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - testFullTextFunctionsCurrentlyUnsupportedBehaviour("multi_match(\"Anna\", first_name)"); + testFullTextFunctionsCurrentlyUnsupportedBehaviour("multi_match(\"Meditation\", title)"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - testFullTextFunctionsCurrentlyUnsupportedBehaviour("term(first_name, \"Anna\")"); + testFullTextFunctionsCurrentlyUnsupportedBehaviour("term(title, \"Meditation\")"); } } private void testFullTextFunctionsCurrentlyUnsupportedBehaviour(String functionInvocation) throws Exception { assertThat( - error("from test | stats max_salary = max(salary) by emp_no | where " + functionInvocation), + error("from test | stats max_salary = max(salary) by emp_no | where " + functionInvocation, fullTextAnalyzer), containsString("Unknown column") ); } public void testFullTextFunctionsNullArgs() throws Exception { testFullTextFunctionNullArgs("match(null, \"query\")", "first"); - testFullTextFunctionNullArgs("match(first_name, null)", "second"); + testFullTextFunctionNullArgs("match(title, null)", "second"); testFullTextFunctionNullArgs("qstr(null)", ""); testFullTextFunctionNullArgs("kql(null)", ""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - testFullTextFunctionNullArgs("multi_match(null, first_name)", "first"); + testFullTextFunctionNullArgs("multi_match(null, title)", "first"); testFullTextFunctionNullArgs("multi_match(\"query\", null)", "second"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { testFullTextFunctionNullArgs("term(null, \"query\")", "first"); - testFullTextFunctionNullArgs("term(first_name, null)", "second"); + testFullTextFunctionNullArgs("term(title, null)", "second"); } } private void testFullTextFunctionNullArgs(String functionInvocation, String argOrdinal) throws Exception { assertThat( - error("from test | where " + functionInvocation), + error("from test | where " + functionInvocation, fullTextAnalyzer), containsString(argOrdinal + " argument of [" + functionInvocation + "] cannot be null, received [null]") ); } public void testFullTextFunctionsConstantQuery() throws Exception { - testFullTextFunctionsConstantQuery("match(first_name, last_name)", "second"); - testFullTextFunctionsConstantQuery("qstr(first_name)", ""); - testFullTextFunctionsConstantQuery("kql(first_name)", ""); + testFullTextFunctionsConstantQuery("match(title, category)", "second"); + testFullTextFunctionsConstantQuery("qstr(title)", ""); + testFullTextFunctionsConstantQuery("kql(title)", ""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - testFullTextFunctionsConstantQuery("multi_match(first_name, first_name)", "first"); - testFullTextFunctionsConstantQuery("multi_match(concat(first_name, \"world\"), first_name)", "first"); + testFullTextFunctionsConstantQuery("multi_match(category, body)", "first"); + testFullTextFunctionsConstantQuery("multi_match(concat(title, \"world\"), title)", "first"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - testFullTextFunctionsConstantQuery("term(first_name, last_name)", "second"); + testFullTextFunctionsConstantQuery("term(title, tags)", "second"); } } private void testFullTextFunctionsConstantQuery(String functionInvocation, String argOrdinal) throws Exception { assertThat( - error("from test | where " + functionInvocation), + error("from test | where " + functionInvocation, fullTextAnalyzer), containsString(argOrdinal + " argument of [" + functionInvocation + "] must be a constant") ); } @@ -2156,23 +2162,23 @@ public void testInsistNotOnTopOfFrom() { } public void testFullTextFunctionsInStats() { - checkFullTextFunctionsInStats("match(last_name, \"Smith\")"); - checkFullTextFunctionsInStats("last_name : \"Smith\""); - checkFullTextFunctionsInStats("qstr(\"last_name: Smith\")"); - checkFullTextFunctionsInStats("kql(\"last_name: Smith\")"); + checkFullTextFunctionsInStats("match(title, \"Meditation\")"); + checkFullTextFunctionsInStats("title : \"Meditation\""); + checkFullTextFunctionsInStats("qstr(\"title: Meditation\")"); + checkFullTextFunctionsInStats("kql(\"title: Meditation\")"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - checkFullTextFunctionsInStats("multi_match(\"Smith\", first_name, last_name)"); + checkFullTextFunctionsInStats("multi_match(\"Meditation\", title, body)"); } } private void checkFullTextFunctionsInStats(String functionInvocation) { - query("from test | stats c = max(salary) where " + functionInvocation); - query("from test | stats c = max(salary) where " + functionInvocation + " or length(first_name) > 10"); - query("from test metadata _score | where " + functionInvocation + " | stats c = max(_score)"); - query("from test metadata _score | where " + functionInvocation + " or length(first_name) > 10 | stats c = max(_score)"); + query("from test | stats c = max(id) where " + functionInvocation, fullTextAnalyzer); + query("from test | stats c = max(id) where " + functionInvocation + " or length(title) > 10", fullTextAnalyzer); + query("from test metadata _score | where " + functionInvocation + " | stats c = max(_score)", fullTextAnalyzer); + query("from test metadata _score | where " + functionInvocation + " or length(title) > 10 | stats c = max(_score)", fullTextAnalyzer); assertThat( - error("from test metadata _score | stats c = max(_score) where " + functionInvocation), + error("from test metadata _score | stats c = max(_score) where " + functionInvocation, fullTextAnalyzer), containsString("cannot use _score aggregations with a WHERE filter in a STATS command") ); } From 1774cb0bfa5a342bee8b9b28735cbd3ed92ecc07 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Tue, 3 Jun 2025 12:07:13 +0200 Subject: [PATCH 09/14] Don't use data with the CSV data loader, just the mapping --- .../xpack/esql/CsvTestsDataLoader.java | 4 +--- .../main/resources/data/full_text_search.csv | 21 ------------------- .../src/main/resources/knn-function.csv-spec | 18 ++++++++-------- 3 files changed, 10 insertions(+), 33 deletions(-) delete mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/full_text_search.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 66fca56efc1a8..c041fe55c32fc 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -145,7 +145,6 @@ public class CsvTestsDataLoader { private static final TestDataset MV_TEXT = new TestDataset("mv_text"); private static final TestDataset DENSE_VECTOR = new TestDataset("dense_vector"); private static final TestDataset COLORS = new TestDataset("colors"); - private static final TestDataset FULL_TEXT_SEARCH = new TestDataset("full_text_search"); public static final Map CSV_DATASET_MAP = Map.ofEntries( Map.entry(EMPLOYEES.indexName, EMPLOYEES), @@ -207,8 +206,7 @@ public class CsvTestsDataLoader { Map.entry(LOGS.indexName, LOGS), Map.entry(MV_TEXT.indexName, MV_TEXT), Map.entry(DENSE_VECTOR.indexName, DENSE_VECTOR), - Map.entry(COLORS.indexName, COLORS), - Map.entry(FULL_TEXT_SEARCH.indexName, FULL_TEXT_SEARCH) + Map.entry(COLORS.indexName, COLORS) ); private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json"); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/full_text_search.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/full_text_search.csv deleted file mode 100644 index 8d0fecdb4f8aa..0000000000000 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/full_text_search.csv +++ /dev/null @@ -1,21 +0,0 @@ -id:integer,title:text,body:text,tags:keyword,category:keyword,published_date:date,vector:dense_vector -1,The Rise of AI,Artificial intelligence is revolutionizing industries, from healthcare to finance.,ai,technology,future,technology,2023-01-15,[0.89, 0.61, 0.13] -2,Hiking the Grand Canyon,Exploring the vast landscapes of the Grand Canyon is an unforgettable experience.,travel,nature,hiking,travel,2022-11-20,[0.31, 0.85, 0.44] -3,Understanding Quantum Computing,Quantum computing leverages the principles of quantum mechanics for computation.,quantum,computing,research,science,2023-06-05,[0.92, 0.47, 0.22] -4,Healthy Meal Planning,Meal prepping with nutritious ingredients can save time and improve well-being.,health,food,planning,lifestyle,2024-03-08,[0.41, 0.66, 0.30] -5,Dogs: Loyal Companions,Dogs provide emotional support and are known for their loyalty and affection.,dogs,pets,companionship,animals,2021-12-30,[0.14, 0.91, 0.19] -6,A Guide to the Solar System,The solar system is home to eight planets, each with unique characteristics.,space,planets,astronomy,science,2022-07-14,[0.75, 0.34, 0.56] -7,Meditation for Beginners,Meditation can help reduce stress and improve mental clarity when practiced regularly.,meditation,wellness,mental health,lifestyle,2023-02-28,[0.36, 0.72, 0.28] -8,Exploring Tokyo,Tokyo blends modern skyscrapers with traditional temples and vibrant street culture.,japan,tokyo,travel,travel,2024-09-10,[0.45, 0.82, 0.39] -9,Introduction to Neural Networks,Neural networks are foundational to deep learning, a subfield of machine learning.,neural networks,deep learning,ai,technology,2023-10-01,[0.88, 0.60, 0.15] -10,Gardening Tips for Spring,Spring is the ideal time to start planting flowers, herbs, and vegetables.,gardening,plants,spring,lifestyle,2022-03-15,[0.33, 0.76, 0.21] -11,Basics of Blockchain Technology,Blockchain provides a decentralized way to store and verify transactions.,blockchain,cryptocurrency,tech,technology,2023-05-22,[0.91, 0.55, 0.20] -12,Cats vs Dogs,Cats and dogs are both popular pets, each with unique behaviors and needs.,cats,dogs,pets,animals,2022-08-18,[0.18, 0.89, 0.23] -13,The Benefits of Yoga,Yoga combines physical postures with breathing exercises and meditation.,yoga,fitness,health,lifestyle,2024-01-04,[0.40, 0.69, 0.27] -14,Visiting the Louvre Museum,The Louvre houses famous artworks like the Mona Lisa and Venus de Milo.,art,museum,paris,travel,2021-06-25,[0.50, 0.78, 0.41] -15,Climate Change Explained,Climate change is a global challenge affecting ecosystems and weather patterns.,climate,environment,science,science,2022-10-11,[0.66, 0.70, 0.45] -16,Intro to Programming with Python,Python is a versatile language for beginners and experts alike.,python,programming,code,technology,2023-11-30,[0.87, 0.64, 0.18] -17,Exploring the Amazon Rainforest,The Amazon Rainforest is rich in biodiversity and cultural heritage.,amazon,nature,exploration,travel,2023-07-19,[0.38, 0.84, 0.46] -18,The Basics of Nutrition,Balanced nutrition is essential for energy, growth, and health maintenance.,nutrition,food,health,lifestyle,2022-05-12,[0.43, 0.68, 0.29] -19,Mars Missions Update,NASA and private companies continue exploring Mars with rovers and future plans.,mars,space,exploration,science,2024-04-26,[0.79, 0.37, 0.52] -20,Bird Watching in Costa Rica,Costa Rica offers a paradise for bird watchers with hundreds of species.,birds,wildlife,nature,travel,2021-09-09,[0.29, 0.83, 0.38] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec index de446123c4fcf..be71373d63e23 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec @@ -235,15 +235,15 @@ from colors metadata _score | keep color, round_score ; -color:text | round_score: double -olive | 1.0 -olive drab | 0.0014 -dark olive green | 4.0E-4 -dark golden rod | 3.0E-4 -sienna | 3.0E-4 -medium aqua marine | 0.0 -medium spring green | 0.0 -light golden rod yellow | 0.0 +color:text | round_score:double +olive | 1.0 +olive drab | 0.0014 +dark olive green | 4.0E-4 +dark golden rod | 3.0E-4 +sienna | 3.0E-4 +light golden rod yellow | 0.0 +medium aqua marine | 0.0 +medium spring green | 0.0 ; testKnnWithNonPushableDisjunctionsOnComplexExpressions From 0742884d5bf35c1d455594fd980f8b64d2989b76 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Tue, 3 Jun 2025 12:07:36 +0200 Subject: [PATCH 10/14] Spotless --- .../xpack/esql/analysis/VerifierTests.java | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 4e9c2fddba32e..6c006d672d8a0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1340,7 +1340,10 @@ public void testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands(String fun containsString("[" + functionName + "] function cannot be used after KEEP") ); assertThat( - error("from test | STATS c = COUNT(id) BY category | rename c as total_categories | where " + functionInvocation, fullTextAnalyzer), + error( + "from test | STATS c = COUNT(id) BY category | rename c as total_categories | where " + functionInvocation, + fullTextAnalyzer + ), containsString("[" + functionName + "] function cannot be used after RENAME") ); assertThat( @@ -1403,36 +1406,45 @@ private void checkWithFullTextFunctionsDisjunctions(String functionInvocation) { query("from test | where " + functionInvocation + " or length(title) > 10", fullTextAnalyzer); query("from test | where match(title, \"Meditation\") or (" + functionInvocation + " and length(title) > 10)", fullTextAnalyzer); query( - "from test | where (" - + functionInvocation - + " and length(title) > 0) or (match(title, \"Meditation\") and length(title) > 10)", fullTextAnalyzer + "from test | where (" + functionInvocation + " and length(title) > 0) or (match(title, \"Meditation\") and length(title) > 10)", + fullTextAnalyzer ); // Disjunctions with non-pushable functions - no scoring query("from test | where " + functionInvocation + " or length(title) > 10", fullTextAnalyzer); query("from test | where match(title, \"Meditation\") or (" + functionInvocation + " and length(title) > 10)", fullTextAnalyzer); query( - "from test | where (" - + functionInvocation - + " and length(title) > 0) or (match(title, \"Meditation\") and length(title) > 10)", fullTextAnalyzer + "from test | where (" + functionInvocation + " and length(title) > 0) or (match(title, \"Meditation\") and length(title) > 10)", + fullTextAnalyzer ); // Disjunctions with full text functions - no scoring query("from test | where " + functionInvocation + " or match(title, \"Meditation\")", fullTextAnalyzer); query("from test | where " + functionInvocation + " or not match(title, \"Meditation\")", fullTextAnalyzer); query("from test | where (" + functionInvocation + " or match(title, \"Meditation\")) and length(title) > 10", fullTextAnalyzer); - query("from test | where (" + functionInvocation + " or match(title, \"Meditation\")) and match(body, \"Smith\")", fullTextAnalyzer); - query("from test | where " + functionInvocation + " or (match(title, \"Meditation\") and match(body, \"Smith\"))", fullTextAnalyzer); + query( + "from test | where (" + functionInvocation + " or match(title, \"Meditation\")) and match(body, \"Smith\")", + fullTextAnalyzer + ); + query( + "from test | where " + functionInvocation + " or (match(title, \"Meditation\") and match(body, \"Smith\"))", + fullTextAnalyzer + ); // Disjunctions with full text functions - scoring query("from test metadata _score | where " + functionInvocation + " or match(title, \"Meditation\")", fullTextAnalyzer); query("from test metadata _score | where " + functionInvocation + " or not match(title, \"Meditation\")", fullTextAnalyzer); - query("from test metadata _score | where (" + functionInvocation + " or match(title, \"Meditation\")) and length(title) > 10", fullTextAnalyzer); query( - "from test metadata _score | where (" + functionInvocation + " or match(title, \"Meditation\")) and match(body, \"Smith\")", fullTextAnalyzer + "from test metadata _score | where (" + functionInvocation + " or match(title, \"Meditation\")) and length(title) > 10", + fullTextAnalyzer ); query( - "from test metadata _score | where " + functionInvocation + " or (match(title, \"Meditation\") and match(body, \"Smith\"))", fullTextAnalyzer + "from test metadata _score | where (" + functionInvocation + " or match(title, \"Meditation\")) and match(body, \"Smith\")", + fullTextAnalyzer + ); + query( + "from test metadata _score | where " + functionInvocation + " or (match(title, \"Meditation\") and match(body, \"Smith\"))", + fullTextAnalyzer ); } @@ -1517,10 +1529,7 @@ public void testFullTextFunctionsTargetsExistingField() throws Exception { } private void testFullTextFunctionTargetsExistingField(String functionInvocation) throws Exception { - assertThat( - error("from test | keep emp_no | where " + functionInvocation), - containsString("Unknown column") - ); + assertThat(error("from test | keep emp_no | where " + functionInvocation), containsString("Unknown column")); } public void testConditionalFunctionsWithMixedNumericTypes() { @@ -2175,7 +2184,10 @@ private void checkFullTextFunctionsInStats(String functionInvocation) { query("from test | stats c = max(id) where " + functionInvocation, fullTextAnalyzer); query("from test | stats c = max(id) where " + functionInvocation + " or length(title) > 10", fullTextAnalyzer); query("from test metadata _score | where " + functionInvocation + " | stats c = max(_score)", fullTextAnalyzer); - query("from test metadata _score | where " + functionInvocation + " or length(title) > 10 | stats c = max(_score)", fullTextAnalyzer); + query( + "from test metadata _score | where " + functionInvocation + " or length(title) > 10 | stats c = max(_score)", + fullTextAnalyzer + ); assertThat( error("from test metadata _score | stats c = max(_score) where " + functionInvocation, fullTextAnalyzer), From ac0982a9d2d4948cbf631ba1563d6ae84043cec1 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Tue, 3 Jun 2025 12:14:10 +0200 Subject: [PATCH 11/14] Fix naming and visibility of helper methods --- .../xpack/esql/analysis/VerifierTests.java | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 6c006d672d8a0..bf7fde8051e60 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1216,24 +1216,23 @@ public void testMatchInsideEval() throws Exception { } public void testFieldBasedFullTextFunctions() throws Exception { - testFieldBasedWithNonIndexedColumn("MATCH", "match(text, \"cat\")", "function"); - testFieldBasedFunctionNotAllowedAfterCommands("MATCH", "function", "match(title, \"Meditation\")"); + checkFieldBasedWithNonIndexedColumn("MATCH", "match(text, \"cat\")", "function"); + checkFieldBasedFunctionNotAllowedAfterCommands("MATCH", "function", "match(title, \"Meditation\")"); - testFieldBasedWithNonIndexedColumn(":", "text : \"cat\"", "operator"); - testFieldBasedFunctionNotAllowedAfterCommands(":", "operator", "title : \"Meditation\""); + checkFieldBasedWithNonIndexedColumn(":", "text : \"cat\"", "operator"); + checkFieldBasedFunctionNotAllowedAfterCommands(":", "operator", "title : \"Meditation\""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - testFieldBasedWithNonIndexedColumn("MultiMatch", "multi_match(\"cat\", text)", "function"); - testFieldBasedFunctionNotAllowedAfterCommands("MultiMatch", "function", "multi_match(\"Meditation\", title)"); + checkFieldBasedWithNonIndexedColumn("MultiMatch", "multi_match(\"cat\", text)", "function"); + checkFieldBasedFunctionNotAllowedAfterCommands("MultiMatch", "function", "multi_match(\"Meditation\", title)"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - testFieldBasedWithNonIndexedColumn("Term", "term(text, \"cat\")", "function"); - testFieldBasedFunctionNotAllowedAfterCommands("Term", "function", "term(title, \"Meditation\")"); + checkFieldBasedWithNonIndexedColumn("Term", "term(text, \"cat\")", "function"); + checkFieldBasedFunctionNotAllowedAfterCommands("Term", "function", "term(title, \"Meditation\")"); } } - public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, String functionType, String functionInvocation) - throws Exception { + private void checkFieldBasedFunctionNotAllowedAfterCommands(String functionName, String functionType, String functionInvocation) { assertThat( error("from test | limit 10 | where " + functionInvocation, fullTextAnalyzer), containsString("[" + functionName + "] " + functionType + " cannot be used after LIMIT") @@ -1246,7 +1245,7 @@ public void testFieldBasedFunctionNotAllowedAfterCommands(String functionName, S } // These should pass eventually once we lift some restrictions on match function - public void testFieldBasedWithNonIndexedColumn(String functionName, String functionInvocation, String functionType) { + private void checkFieldBasedWithNonIndexedColumn(String functionName, String functionInvocation, String functionType) { assertThat( error("from test | eval text = substring(title, 1) | where " + functionInvocation, fullTextAnalyzer), containsString( @@ -1265,15 +1264,12 @@ public void testFieldBasedWithNonIndexedColumn(String functionName, String funct assertThat(keywordError, containsString("which is not a field from an index mapping")); } - public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { - testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("QSTR", "qstr(\"field_name: Meditation\")"); - } - - public void testKqlFunctionsNotAllowedAfterCommands() throws Exception { - testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("KQL", "kql(\"field_name: Meditation\")"); + public void testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands() throws Exception { + checkNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("QSTR", "qstr(\"field_name: Meditation\")"); + checkNonFieldBasedFullTextFunctionsNotAllowedAfterCommands("KQL", "kql(\"field_name: Meditation\")"); } - public void testNonFieldBasedFullTextFunctionsNotAllowedAfterCommands(String functionName, String functionInvocation) throws Exception { + private void checkNonFieldBasedFullTextFunctionsNotAllowedAfterCommands(String functionName, String functionInvocation) { // Source commands assertThat( error("show info | where " + functionInvocation), From f3da4daf1c5bc886125cf89e663f3acf4aae7c81 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Tue, 3 Jun 2025 12:16:20 +0200 Subject: [PATCH 12/14] Fix merge --- .../src/main/resources/knn-function.csv-spec | 321 ------------------ 1 file changed, 321 deletions(-) delete mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec deleted file mode 100644 index be71373d63e23..0000000000000 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec +++ /dev/null @@ -1,321 +0,0 @@ -knnSearch -required_capability: knn_function - -// tag::knn-function[] -from colors metadata _score -| where knn(rgb_vector, [0, 120, 0]) -| sort _score desc -// end::knn-function[] -| keep color, rgb_vector -; - -// tag::knn-function-result[] -color:text | rgb_vector:dense_vector -green | [0.0, 128.0, 0.0] -dark green | [0.0, 100.0, 0.0] -forest green | [34.0, 139.0, 34.0] -dark olive green | [85.0, 107.0, 47.0] -sea green | [46.0, 139.0, 87.0] -dark slate gray | [47.0, 79.0, 79.0] -olive drab | [107.0, 142.0, 35.0] -lime green | [50.0, 205.0, 50.0] -black | [0.0, 0.0, 0.0] -olive | [128.0, 128.0, 0.0] -// end::knn-function-result[] -; - -knnSearchWithKOption -required_capability: knn_function - -// tag::knn-function-options[] -from colors metadata _score -| where knn(rgb_vector, [0,255,255], {"k": 4}) -| sort _score desc -// end::knn-function-options[] -| keep color, rgb_vector -; - -color:text | rgb_vector:dense_vector -cyan | [0.0, 255.0, 255.0] -deep sky blue | [0.0, 191.0, 255.0] -dark turquoise | [0.0, 206.0, 209.0] -turquoise | [64.0, 224.0, 208.0] -; - -knnSearchWithSimilarityOption -required_capability: knn_function - -from colors metadata _score -| where knn(rgb_vector, [255,192,203], {"similarity": 40}) -| sort _score desc -| keep color, rgb_vector -; - -color:text | rgb_vector:dense_vector -pink | [255.0, 192.0, 203.0] -light pink | [255.0, 182.0, 193.0] -peach puff | [255.0, 218.0, 185.0] -bisque | [255.0, 228.0, 196.0] -thistle | [216.0, 191.0, 216.0] -wheat | [245.0, 222.0, 179.0] -; - -knnHybridSearch -required_capability: knn_function - -from colors metadata _score -| where match(color, "violet") or knn(rgb_vector, [238,130,238], {"boost": 10.0, "k": 5}) -| sort _score desc -| eval round_score = round(_score, 4) -| keep color, rgb_vector, round_score -; - -color:text | rgb_vector:dense_vector | round_score:double -violet | [238.0, 130.0, 238.0] | 13.9457 -blue violet | [138.0, 43.0, 226.0] | 3.0871 -dark violet | [148.0, 0.0, 211.0] | 3.0871 -medium violet red | [199.0, 21.0, 133.0] | 2.5355 -pale violet red | [219.0, 112.0, 147.0] | 2.5355 -orchid | [218.0, 112.0, 214.0] | 0.0083 -plum | [221.0, 160.0, 221.0] | 0.0071 -hot pink | [255.0, 105.0, 180.0] | 0.0024 -thistle | [216.0, 191.0, 216.0] | 0.0021 -; - -knnWithMultipleFunctions -required_capability: knn_function - -from colors metadata _score -| where knn(rgb_vector, [128,128,0]) and match(color, "olive") -| sort _score desc -| eval round_score = round(_score, 4) -| keep color, rgb_vector, round_score -; - -color:text | rgb_vector:dense_vector | round_score:double -olive | [128.0, 128.0, 0.0] | 5.4979 -olive drab | [107.0, 142.0, 35.0] | 3.5206 -dark olive green | [85.0, 107.0, 47.0] | 2.8906 -; - -knnAfterKeep -required_capability: knn_function - -from colors metadata _score -| keep rgb_vector, _score -| where knn(rgb_vector, [128,128,0]) -| eval round_score = round(_score, 4) -| sort round_score desc -| keep rgb_vector, round_score -| limit 5 -; - -rgb_vector:dense_vector | round_score:double -[128.0, 128.0, 0.0] | 1.0 -[107.0, 142.0, 35.0] | 0.0014 -[85.0, 107.0, 47.0] | 4.0E-4 -[139.0, 69.0, 19.0] | 3.0E-4 -[184.0, 134.0, 11.0] | 3.0E-4 -; - -knnAfterDrop -required_capability: knn_function - -from colors metadata _score -| drop color -| where knn(rgb_vector, [128,128,0]) -| eval round_score = round(_score, 4) -| keep rgb_vector, round_score -| limit 5 -; - -rgb_vector:dense_vector | round_score:double -[184.0, 134.0, 11.0] | 3.0E-4 -[128.0, 128.0, 0.0] | 1.0 -[154.0, 205.0, 50.0] | 1.0E-4 -[85.0, 107.0, 47.0] | 4.0E-4 -[107.0, 142.0, 35.0] | 0.0014 -; - -knnAfterEval -required_capability: knn_function - -from colors metadata _score -| eval composed_name = locate(color, " ") > 0 -| where knn(rgb_vector, [128,128,0]) -| sort _score, color desc -| keep color, composed_name -; - -color:text | composed_name:boolean -peru | false -yellow green | true -chocolate | false -dim gray | true -saddle brown | true -sienna | false -dark golden rod | true -dark olive green | true -olive drab | true -olive | false -; - -knnWithConjunction -required_capability: knn_function - -# TODO We need kNN prefiltering here so we get more candidates that pass the filter -from colors metadata _score -| where knn(rgb_vector, [255,255,238]) and hex_code like "#FFF*" -| keep color, hex_code, rgb_vector -; -ignoreOrder:true - -color:text | hex_code: keyword | rgb_vector:dense_vector -light yellow | #FFFFE0 | [255.0, 255.0, 224.0] -lavender blush | #FFF0F5 | [255.0, 240.0, 245.0] -sea shell | #FFF5EE | [255.0, 245.0, 238.0] -floral white | #FFFAF0 | [255.0, 250.0, 240.0] -ivory | #FFFFF0 | [255.0, 255.0, 240.0] -snow | #FFFAFA | [255.0, 250.0, 250.0] -white | #FFFFFF | [255.0, 255.0, 255.0] -; - -knnWithDisjunctionAndFiltersConjunction -required_capability: knn_function - -# TODO We need kNN prefiltering here so we get more candidates that pass the filter -from colors metadata _score -| where (knn(rgb_vector, [0,255,255]) or knn(rgb_vector, [128, 0, 255])) and primary == true -| keep color, rgb_vector, _score -; - -color:text | rgb_vector:dense_vector | _score:double -cyan | [0.0, 255.0, 255.0] | 1.0 -blue | [0.0, 0.0, 255.0] | 9.922293975250795E-5 -; - -knnWithDisjunctionAndConjunction -required_capability: knn_function -required_capability: full_text_functions_disjunctions - -# TODO We need kNN prefiltering here so we get more candidates that pass the filter -from colors metadata _score -| where (knn(rgb_vector, [0,255,255]) or knn(rgb_vector, [0, 0, 255])) and knn(rgb_vector, [0, 255, 0]) -| keep color, rgb_vector, _score -; - -color:text | rgb_vector:dense_vector | _score:double -medium spring green | [0.0, 250.0, 154.0] | 1.6871128173079342E-4 -; - -knnWithNonPushableConjunction -required_capability: knn_function - -from colors metadata _score -| eval composed_name = locate(color, " ") > 0 -| where knn(rgb_vector, [128,128,0]) and composed_name == false -| eval round_score = round(_score, 4) -| keep color, composed_name, round_score -; - -color:text | composed_name:boolean | round_score:double -olive | false | 1.0 -sienna | false | 3.0E-4 -chocolate | false | 1.0E-4 -peru | false | 1.0E-4 -; - -testKnnWithNonPushableDisjunctions -required_capability: knn_function - -from colors metadata _score -| where knn(rgb_vector, [128,128,0], {"k": 5}) or length(color) > 17 -| sort _score desc, color asc -| eval round_score = round(_score, 4) -| keep color, round_score -; - -color:text | round_score:double -olive | 1.0 -olive drab | 0.0014 -dark olive green | 4.0E-4 -dark golden rod | 3.0E-4 -sienna | 3.0E-4 -light golden rod yellow | 0.0 -medium aqua marine | 0.0 -medium spring green | 0.0 -; - -testKnnWithNonPushableDisjunctionsOnComplexExpressions -required_capability: knn_function - -from colors metadata _score -| where (knn(rgb_vector, [128,128,0]) and length(color) > 12) or (knn(rgb_vector, [128,0,128]) and primary == false) -| sort _score desc -| eval round_score = round(_score, 4) -| keep color, primary, round_score -; - -color: text | primary: boolean | round_score: double -purple | false | 1.0 -dark magenta | false | 0.0045 -dark olive green | false | 4.0E-4 -indigo | false | 4.0E-4 -dark golden rod | false | 3.0E-4 -dim gray | false | 3.0E-4 -dark slate blue | false | 2.0E-4 -medium violet red | false | 2.0E-4 -dark orchid | false | 1.0E-4 -dark violet | false | 1.0E-4 -brown | false | 1.0E-4 -blue violet | false | 1.0E-4 -; - -testKnnInStatsNonPushable -required_capability: knn_function - -from colors -| where length(color) < 10 -| stats c = count(*) where knn(rgb_vector, [128,128,255], {"k": 140}) -; - -c: long -59 -; - - -testKnnInStatsPushableAndNonPushable -required_capability: knn_function -required_capability: full_text_functions_in_stats_where - -from colors metadata _score -| stats c = count(*) where (knn(rgb_vector, [0,255,255], {"k": 140}) or knn(rgb_vector, [0, 0, 255])) and knn(rgb_vector, [0, 255, 0], {"k": 40}) -; - -c:long -40 -; - -testKnnInStatsWithGrouping -from colors -| where length(color) < 10 -| stats c = count(*) where knn(rgb_vector, [128,128,255], {"k": 140}) by primary -; - -c: long | primary: boolean -50 | false -9 | true -; - -testKnnInStatsPushable -required_capability: knn_function -required_capability: full_text_functions_in_stats_where - -from colors -| stats c = count(*) where knn(rgb_vector, [128,128,255], {"k": 40}) -; - -# No surprises, gets the number of top k -c:long -40 -; From 28182982c902523fe855ad6c181cfa9f0883bb62 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Tue, 3 Jun 2025 12:54:15 +0200 Subject: [PATCH 13/14] Fix naming of helper methods --- .../xpack/esql/analysis/VerifierTests.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index bf7fde8051e60..1d5de414f4c17 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -2116,21 +2116,21 @@ private void testFullTextFunctionsCurrentlyUnsupportedBehaviour(String functionI } public void testFullTextFunctionsNullArgs() throws Exception { - testFullTextFunctionNullArgs("match(null, \"query\")", "first"); - testFullTextFunctionNullArgs("match(title, null)", "second"); - testFullTextFunctionNullArgs("qstr(null)", ""); - testFullTextFunctionNullArgs("kql(null)", ""); + checkFullTextFunctionNullArgs("match(null, \"query\")", "first"); + checkFullTextFunctionNullArgs("match(title, null)", "second"); + checkFullTextFunctionNullArgs("qstr(null)", ""); + checkFullTextFunctionNullArgs("kql(null)", ""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - testFullTextFunctionNullArgs("multi_match(null, title)", "first"); - testFullTextFunctionNullArgs("multi_match(\"query\", null)", "second"); + checkFullTextFunctionNullArgs("multi_match(null, title)", "first"); + checkFullTextFunctionNullArgs("multi_match(\"query\", null)", "second"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - testFullTextFunctionNullArgs("term(null, \"query\")", "first"); - testFullTextFunctionNullArgs("term(title, null)", "second"); + checkFullTextFunctionNullArgs("term(null, \"query\")", "first"); + checkFullTextFunctionNullArgs("term(title, null)", "second"); } } - private void testFullTextFunctionNullArgs(String functionInvocation, String argOrdinal) throws Exception { + private void checkFullTextFunctionNullArgs(String functionInvocation, String argOrdinal) throws Exception { assertThat( error("from test | where " + functionInvocation, fullTextAnalyzer), containsString(argOrdinal + " argument of [" + functionInvocation + "] cannot be null, received [null]") @@ -2138,19 +2138,19 @@ private void testFullTextFunctionNullArgs(String functionInvocation, String argO } public void testFullTextFunctionsConstantQuery() throws Exception { - testFullTextFunctionsConstantQuery("match(title, category)", "second"); - testFullTextFunctionsConstantQuery("qstr(title)", ""); - testFullTextFunctionsConstantQuery("kql(title)", ""); + checkFullTextFunctionsConstantQuery("match(title, category)", "second"); + checkFullTextFunctionsConstantQuery("qstr(title)", ""); + checkFullTextFunctionsConstantQuery("kql(title)", ""); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { - testFullTextFunctionsConstantQuery("multi_match(category, body)", "first"); - testFullTextFunctionsConstantQuery("multi_match(concat(title, \"world\"), title)", "first"); + checkFullTextFunctionsConstantQuery("multi_match(category, body)", "first"); + checkFullTextFunctionsConstantQuery("multi_match(concat(title, \"world\"), title)", "first"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { - testFullTextFunctionsConstantQuery("term(title, tags)", "second"); + checkFullTextFunctionsConstantQuery("term(title, tags)", "second"); } } - private void testFullTextFunctionsConstantQuery(String functionInvocation, String argOrdinal) throws Exception { + private void checkFullTextFunctionsConstantQuery(String functionInvocation, String argOrdinal) throws Exception { assertThat( error("from test | where " + functionInvocation, fullTextAnalyzer), containsString(argOrdinal + " argument of [" + functionInvocation + "] must be a constant") From ede0896c13e8d754f5b2a66ff3a3d1f8747e7b4c Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Tue, 3 Jun 2025 13:48:08 +0200 Subject: [PATCH 14/14] Remove colors dataset from knn --- .../java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index c041fe55c32fc..92fe597362bb0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -144,7 +144,6 @@ public class CsvTestsDataLoader { private static final TestDataset LOGS = new TestDataset("logs"); private static final TestDataset MV_TEXT = new TestDataset("mv_text"); private static final TestDataset DENSE_VECTOR = new TestDataset("dense_vector"); - private static final TestDataset COLORS = new TestDataset("colors"); public static final Map CSV_DATASET_MAP = Map.ofEntries( Map.entry(EMPLOYEES.indexName, EMPLOYEES), @@ -205,8 +204,7 @@ public class CsvTestsDataLoader { Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT), Map.entry(LOGS.indexName, LOGS), Map.entry(MV_TEXT.indexName, MV_TEXT), - Map.entry(DENSE_VECTOR.indexName, DENSE_VECTOR), - Map.entry(COLORS.indexName, COLORS) + Map.entry(DENSE_VECTOR.indexName, DENSE_VECTOR) ); private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json");