From f33b9949382ad7519e1189596bc8e8a9a57d7c70 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Mon, 5 May 2025 00:10:27 +0800 Subject: [PATCH 1/9] Disallow removal of regex extracted fields --- .../org/elasticsearch/xpack/esql/session/EsqlSession.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 7312ddb7da790..6cd9c92c606c2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -600,9 +600,11 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy parsed.forEachDown(p -> {// go over each plan top-down if (p instanceof RegexExtract re) { // for Grok and Dissect - // remove other down-the-tree references to the extracted fields - for (Attribute extracted : re.extractedFields()) { - referencesBuilder.removeIf(attr -> matchByName(attr, extracted.name(), false)); + if (canRemoveAliases[0]) { + // remove other down-the-tree references to the extracted fields + for (Attribute extracted : re.extractedFields()) { + referencesBuilder.removeIf(attr -> matchByName(attr, extracted.name(), false)); + } } // but keep the inputs needed by Grok/Dissect referencesBuilder.addAll(re.input().references()); From b42cf9f00f1c7dcd5fd6d29ebb6f46e893ce9212 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Mon, 5 May 2025 00:48:26 +0800 Subject: [PATCH 2/9] Update docs/changelog/127687.yaml --- docs/changelog/127687.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/127687.yaml diff --git a/docs/changelog/127687.yaml b/docs/changelog/127687.yaml new file mode 100644 index 0000000000000..e053c4a31ad2e --- /dev/null +++ b/docs/changelog/127687.yaml @@ -0,0 +1,6 @@ +pr: 127687 +summary: "ESQL: Fix alias removal in regex extraction with JOIN" +area: ES|QL +type: bug +issues: + - 127467 From 0655294ffbc30931eb77177e37e6abb034321ceb Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Mon, 5 May 2025 13:30:23 +0800 Subject: [PATCH 3/9] Update tests --- .../rest/generative/GenerativeRestTest.java | 1 - .../src/main/resources/lookup-join.csv-spec | 66 +++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 6 ++ .../session/IndexResolverFieldNamesTests.java | 22 +++++++ 4 files changed, 94 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 4116344de6759..0bf0e233cd84f 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -54,7 +54,6 @@ public abstract class GenerativeRestTest extends ESRestTestCase { // https://github.com/elastic/elasticsearch/issues/127167 "optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/116781 "No matches found for pattern", // https://github.com/elastic/elasticsearch/issues/126418 - "Unknown column", // https://github.com/elastic/elasticsearch/issues/127467 "only supports KEYWORD or TEXT values", // https://github.com/elastic/elasticsearch/issues/127468 "The incoming YAML document exceeds the limit:" // still to investigate, but it seems to be specific to the test framework ); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index b9c65707146ff..7939da9a67a4a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1650,3 +1650,69 @@ event_duration:long 2764889 3450233 ; + + +joinMaskingRegex +required_capability: union_types +required_capability: join_lookup_v12 +required_capability: fix_join_masking_regex_extract +from books,message_*,ul* +| enrich languages_policy on status +| drop `language_name`, `bytes_out`, `id`, id +| dissect book_no "%{type}" +| dissect author.keyword "%{HZicfARaID}" +| mv_expand `status` +| sort HZicfARaID, year DESC NULLS LAST, publisher DESC NULLS FIRST, description DESC, type NULLS LAST, message ASC NULLS LAST, title NULLS FIRST, status NULLS LAST +| enrich languages_policy on book_no +| grok message "%{WORD:DiLNyZKNDu}" +| limit 7972 +| rename year as language_code +| lookup join languages_lookup on language_code +| limit 13966 +| stats rcyIZnSOb = min(language_code), `ratings` = min(@timestamp), dgDxwMeFYrD = count(`@timestamp`), ifyZfXigqVN = count(*), qTXdrzSpY = min(language_code) by author.keyword +| rename author.keyword as message +| lookup join message_types_lookup on message +| stats `ratings` = count(*) by type +| stats `type` = count(type), `ratings` = count(*) +| keep `ratings`, ratings +; + +ratings:long +1 +; + +joinMaskingDissect +required_capability: join_lookup_v12 +required_capability: fix_join_masking_regex_extract +from sample_data +| dissect message "%{type}" +| drop type +| lookup join message_types_lookup on message +| stats count = count(*) by type +| keep count +| sort count +; +count:long +1 +3 +3 +; + + +joinMaskingGrok +required_capability: join_lookup_v12 +required_capability: fix_join_masking_regex_extract +from sample_data +| grok message "%{WORD:type}" +| drop type +| lookup join message_types_lookup on message +| stats max = max(event_duration) by type +| keep max +| sort max +; + +max:long +1232382 +3450233 +8268153 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 7535b6fc4a010..bcc9c9df492e3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1036,6 +1036,12 @@ public enum Cap { */ FIX_JOIN_MASKING_EVAL, + /** + * During resolution (pre-analysis) we have to consider that joins can override regex extracted values + * see ES|QL: pruning of JOINs leads to missing fields #127467 + */ + FIX_JOIN_MASKING_REGEX_EXTRACT, + /** * Support last_over_time aggregation that gets evaluated per time-series */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index ef54dec2a0b60..8637aa42b7a94 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1710,6 +1710,28 @@ public void testEnrichAndJoinMaskingEvalWh() { | keep emp_no, language_name""", Set.of("emp_no", "language_name", "languages", "language_name.*", "languages.*", "emp_no.*")); } + public void testJoinMaskingGrok() { + assertFieldNames(""" + from message_types + | grok message "%{WORD:type}" + | drop type + | lookup join message_types_lookup on message + | stats `ratings` = count(*) by type + | keep ratings + """, Set.of("type", "message", "ratings", "message.*", "type.*", "ratings.*")); + } + + public void testJoinMaskingDissect() { + assertFieldNames(""" + from message_types + | dissect message "%{type}" + | stats message = max(message) + | lookup join message_types_lookup on message + | stats `ratings` = count(*) by type + | keep ratings + """, Set.of("type", "message", "ratings", "message.*", "type.*", "ratings.*")); + } + private Set fieldNames(String query, Set enrichPolicyMatchFields) { var preAnalysisResult = new EsqlSession.PreAnalysisResult(null); return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames(); From ed1074ea1beb7b09549bfe434d885b06072d8f38 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 9 May 2025 00:01:21 +0800 Subject: [PATCH 4/9] Also remove `ReferenceAttribute` --- .../xpack/esql/session/EsqlSession.java | 22 ++++++------ .../session/IndexResolverFieldNamesTests.java | 34 +++++++------------ 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 6cd9c92c606c2..d7c75d35861e4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -41,6 +41,8 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar; import org.elasticsearch.xpack.esql.core.util.Holder; @@ -600,13 +602,7 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy parsed.forEachDown(p -> {// go over each plan top-down if (p instanceof RegexExtract re) { // for Grok and Dissect - if (canRemoveAliases[0]) { - // remove other down-the-tree references to the extracted fields - for (Attribute extracted : re.extractedFields()) { - referencesBuilder.removeIf(attr -> matchByName(attr, extracted.name(), false)); - } - } - // but keep the inputs needed by Grok/Dissect + // keep the inputs needed by Grok/Dissect referencesBuilder.addAll(re.input().references()); } else if (p instanceof Enrich enrich) { AttributeSet enrichFieldRefs = Expressions.references(enrich.enrichFields()); @@ -663,15 +659,21 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy // remove any already discovered UnresolvedAttributes that are in fact aliases defined later down in the tree // for example "from test | eval x = salary | stats max = max(x) by gender" // remove the UnresolvedAttribute "x", since that is an Alias defined in "eval" + // also remove other down-the-tree references to the extracted fields from "grok" and "dissect" AttributeSet planRefs = p.references(); Set fieldNames = planRefs.names(); - p.forEachExpressionDown(Alias.class, alias -> { + p.forEachExpressionDown(NamedExpression.class, expression -> { + if ((expression instanceof Alias || expression instanceof ReferenceAttribute) == false) { + return; + } // do not remove the UnresolvedAttribute that has the same name as its alias, ie "rename id AS id" // or the UnresolvedAttributes that are used in Functions that have aliases "STATS id = MAX(id)" - if (fieldNames.contains(alias.name())) { + if (fieldNames.contains(expression.name())) { return; } - referencesBuilder.removeIf(attr -> matchByName(attr, alias.name(), keepCommandRefsBuilder.contains(attr))); + referencesBuilder.removeIf( + attr -> matchByName(attr, expression.name(), (expression instanceof Alias) && keepCommandRefsBuilder.contains(attr)) + ); }); } }); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 8637aa42b7a94..0ebdb170b9eec 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1341,6 +1341,18 @@ public void testDissectOverwriteName() { assertThat(fieldNames, equalTo(Set.of("emp_no", "emp_no.*", "first_name", "first_name.*"))); } + public void testAvoidGrokAttributesRemoval() { + Set fieldNames = fieldNames(""" + from message_types + | eval type = 1 + | lookup join message_types_lookup on message + | drop message + | grok type "%{WORD:b}" + | stats x = max(b) + | keep x""", Set.of()); + assertThat(fieldNames, equalTo(Set.of("message", "x", "x.*", "message.*"))); + } + public void testEnrichOnDefaultField() { Set fieldNames = fieldNames(""" from employees @@ -1710,28 +1722,6 @@ public void testEnrichAndJoinMaskingEvalWh() { | keep emp_no, language_name""", Set.of("emp_no", "language_name", "languages", "language_name.*", "languages.*", "emp_no.*")); } - public void testJoinMaskingGrok() { - assertFieldNames(""" - from message_types - | grok message "%{WORD:type}" - | drop type - | lookup join message_types_lookup on message - | stats `ratings` = count(*) by type - | keep ratings - """, Set.of("type", "message", "ratings", "message.*", "type.*", "ratings.*")); - } - - public void testJoinMaskingDissect() { - assertFieldNames(""" - from message_types - | dissect message "%{type}" - | stats message = max(message) - | lookup join message_types_lookup on message - | stats `ratings` = count(*) by type - | keep ratings - """, Set.of("type", "message", "ratings", "message.*", "type.*", "ratings.*")); - } - private Set fieldNames(String query, Set enrichPolicyMatchFields) { var preAnalysisResult = new EsqlSession.PreAnalysisResult(null); return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames(); From 6c700a8762aa7fa6a4ce0b67dc93eef34f80917b Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Sat, 10 May 2025 14:11:48 +0800 Subject: [PATCH 5/9] Comments --- .../testFixtures/src/main/resources/lookup-join.csv-spec | 3 +++ .../org/elasticsearch/xpack/esql/session/EsqlSession.java | 8 ++++---- .../xpack/esql/session/IndexResolverFieldNamesTests.java | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 7939da9a67a4a..8b1bf00df4d60 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1653,6 +1653,7 @@ event_duration:long joinMaskingRegex +// https://github.com/elastic/elasticsearch/issues/127467 required_capability: union_types required_capability: join_lookup_v12 required_capability: fix_join_masking_regex_extract @@ -1682,6 +1683,7 @@ ratings:long ; joinMaskingDissect +// https://github.com/elastic/elasticsearch/issues/127467 required_capability: join_lookup_v12 required_capability: fix_join_masking_regex_extract from sample_data @@ -1700,6 +1702,7 @@ count:long joinMaskingGrok +// https://github.com/elastic/elasticsearch/issues/127467 required_capability: join_lookup_v12 required_capability: fix_join_masking_regex_extract from sample_data diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index d7c75d35861e4..e327e119cb319 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -662,17 +662,17 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy // also remove other down-the-tree references to the extracted fields from "grok" and "dissect" AttributeSet planRefs = p.references(); Set fieldNames = planRefs.names(); - p.forEachExpressionDown(NamedExpression.class, expression -> { - if ((expression instanceof Alias || expression instanceof ReferenceAttribute) == false) { + p.forEachExpressionDown(NamedExpression.class, ne -> { + if ((ne instanceof Alias || ne instanceof ReferenceAttribute) == false) { return; } // do not remove the UnresolvedAttribute that has the same name as its alias, ie "rename id AS id" // or the UnresolvedAttributes that are used in Functions that have aliases "STATS id = MAX(id)" - if (fieldNames.contains(expression.name())) { + if (fieldNames.contains(ne.name())) { return; } referencesBuilder.removeIf( - attr -> matchByName(attr, expression.name(), (expression instanceof Alias) && keepCommandRefsBuilder.contains(attr)) + attr -> matchByName(attr, ne.name(), keepCommandRefsBuilder.contains(attr)) ); }); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 0ebdb170b9eec..14c6c93aa1f44 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1341,6 +1341,10 @@ public void testDissectOverwriteName() { assertThat(fieldNames, equalTo(Set.of("emp_no", "emp_no.*", "first_name", "first_name.*"))); } + /** + * Fix alias removal in regex extraction with JOIN + * @see ES|QL: pruning of JOINs leads to missing fields + */ public void testAvoidGrokAttributesRemoval() { Set fieldNames = fieldNames(""" from message_types From 044d3faff7b5e47c15262d69302ea55bf6c90bb1 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Sat, 10 May 2025 14:31:32 +0800 Subject: [PATCH 6/9] Add capability --- .../xpack/esql/session/IndexResolverFieldNamesTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 14c6c93aa1f44..d8ce46972d69d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1346,6 +1346,7 @@ public void testDissectOverwriteName() { * @see ES|QL: pruning of JOINs leads to missing fields */ public void testAvoidGrokAttributesRemoval() { + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); Set fieldNames = fieldNames(""" from message_types | eval type = 1 From 057a8a180da68be3c226dcd155554c1a7e9173f6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 12 May 2025 21:20:42 +0000 Subject: [PATCH 7/9] [CI] Auto commit changes from spotless --- .../org/elasticsearch/xpack/esql/session/EsqlSession.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index e327e119cb319..9d092e79dcc30 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -671,9 +671,7 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy if (fieldNames.contains(ne.name())) { return; } - referencesBuilder.removeIf( - attr -> matchByName(attr, ne.name(), keepCommandRefsBuilder.contains(attr)) - ); + referencesBuilder.removeIf(attr -> matchByName(attr, ne.name(), keepCommandRefsBuilder.contains(attr))); }); } }); From d56383b4f10bf4030becff6c29c5ce84518358ca Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Tue, 13 May 2025 09:15:37 +0800 Subject: [PATCH 8/9] Update `IndexResolverFieldNamesTests` --- .../session/IndexResolverFieldNamesTests.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index d8ce46972d69d..121c3f5558ba7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1358,6 +1358,36 @@ public void testAvoidGrokAttributesRemoval() { assertThat(fieldNames, equalTo(Set.of("message", "x", "x.*", "message.*"))); } + public void testAvoidGrokAttributesRemoval2() { + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + Set fieldNames = fieldNames(""" + from sample_data + | dissect message "%{type}" + | drop type + | lookup join message_types_lookup on message + | stats count = count(*) by type + | keep count + | sort count""", Set.of()); + assertThat(fieldNames, equalTo(Set.of("type", "message", "count", "message.*", "type.*", "count.*"))); + } + + public void testAvoidGrokAttributesRemoval3() { + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + Set fieldNames = fieldNames(""" + from sample_data + | grok message "%{WORD:type}" + | drop type + | lookup join message_types_lookup on message + | stats max = max(event_duration) by type + | keep max + | sort max""", Set.of()); + assertThat( + fieldNames, + equalTo(Set.of("type", "event_duration", "message", "max", "event_duration.*", "message.*", "type.*", "max.*")) + ); + + } + public void testEnrichOnDefaultField() { Set fieldNames = fieldNames(""" from employees From ba02e1d2cac80e33ebbb3cdebc99c69f49b9ad85 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Tue, 20 May 2025 18:12:41 +0800 Subject: [PATCH 9/9] Move capability to the bottom --- .../xpack/esql/action/EsqlCapabilities.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 596952a12e45f..4a45ddeeecba3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1042,12 +1042,6 @@ public enum Cap { */ DROP_AGAIN_WITH_WILDCARD_AFTER_EVAL, - /** - * During resolution (pre-analysis) we have to consider that joins can override regex extracted values - * see ES|QL: pruning of JOINs leads to missing fields #127467 - */ - FIX_JOIN_MASKING_REGEX_EXTRACT, - /** * Support last_over_time aggregation that gets evaluated per time-series */ @@ -1091,7 +1085,13 @@ public enum Cap { /** * Full text functions in STATS */ - FULL_TEXT_FUNCTIONS_IN_STATS_WHERE; + FULL_TEXT_FUNCTIONS_IN_STATS_WHERE, + + /** + * During resolution (pre-analysis) we have to consider that joins can override regex extracted values + * see ES|QL: pruning of JOINs leads to missing fields #127467 + */ + FIX_JOIN_MASKING_REGEX_EXTRACT; private final boolean enabled;