From 1a8dadc7733075c7e2f685023960fdb5ceaf662e Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Thu, 17 Apr 2025 22:09:26 +0800 Subject: [PATCH 1/9] Keep attribute --- .../java/org/elasticsearch/xpack/esql/session/EsqlSession.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index be90e6fd3b3c0..a95e07fa70baf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -626,7 +626,7 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy p.forEachExpression(UnresolvedNamePattern.class, up -> { var ua = new UnresolvedAttribute(up.source(), up.name()); referencesBuilder.add(ua); - if (p instanceof Keep) { + if (p instanceof Keep || p instanceof Drop) { keepCommandRefsBuilder.add(ua); } }); From 424aff5375ec419e586a826f6ee4a1c7b1d44d86 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Thu, 17 Apr 2025 22:43:00 +0800 Subject: [PATCH 2/9] Add tests --- .../testFixtures/src/main/resources/drop.csv-spec | 15 +++++++++++++++ .../session/IndexResolverFieldNamesTests.java | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec index 9886d6cce0ca2..fd9cab7f21e81 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec @@ -172,3 +172,18 @@ Milky Way Milky Way Milky Way ; + +dropAgainWithWildcardAfterEval +from languages +| eval language_code = 12, x = 13 +| drop language_code +| drop language* +| keep x +| limit 3 +; + +x:integer +13 +13 +13 +; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index ef54dec2a0b60..275bfcb011969 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1710,6 +1710,16 @@ public void testEnrichAndJoinMaskingEvalWh() { | keep emp_no, language_name""", Set.of("emp_no", "language_name", "languages", "language_name.*", "languages.*", "emp_no.*")); } + public void testDropAgainWithWildcardAfterEval() { + assertFieldNames(""" + from employees + | eval full_name = 12 + | drop full_name + | drop *name + | keep emp_no + """, Set.of("emp_no", "emp_no.*", "*name", "*name.*")); + } + private Set fieldNames(String query, Set enrichPolicyMatchFields) { var preAnalysisResult = new EsqlSession.PreAnalysisResult(null); return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames(); From 8eca69d81fda235441b3c489233aad5b1b6f9ca5 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Thu, 17 Apr 2025 23:03:06 +0800 Subject: [PATCH 3/9] Update docs/changelog/127009.yaml --- docs/changelog/127009.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/127009.yaml diff --git a/docs/changelog/127009.yaml b/docs/changelog/127009.yaml new file mode 100644 index 0000000000000..adbec139cae28 --- /dev/null +++ b/docs/changelog/127009.yaml @@ -0,0 +1,6 @@ +pr: 127009 +summary: "ESQL: Keep `DROP` attributes when resolving field names" +area: ES|QL +type: bug +issues: + - 126418 From 093933c05b910eec71cd0893c8452f6a90dd218a Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 18 Apr 2025 00:29:40 +0800 Subject: [PATCH 4/9] Remove 126026 from `ALLOWED_ERRORS` --- .../xpack/esql/qa/rest/generative/GenerativeRestTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 8e37c7aaadfbc..73c6437051752 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -54,7 +54,6 @@ public abstract class GenerativeRestTest extends ESRestTestCase { "token recognition error at: '``", // https://github.com/elastic/elasticsearch/issues/125870 "Unknown column \\[.*\\]", // https://github.com/elastic/elasticsearch/issues/126026 "optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/116781 - "No matches found for pattern", // https://github.com/elastic/elasticsearch/issues/126418 "The incoming YAML document exceeds the limit:" // still to investigate, but it seems to be specific to the test framework ); From 7e8c5caebfce76cb6b66ddb321611090f2cb4ceb Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 18 Apr 2025 00:35:44 +0800 Subject: [PATCH 5/9] Fix bwc --- .../esql/qa/testFixtures/src/main/resources/drop.csv-spec | 1 + .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec index fd9cab7f21e81..51597abb062d4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec @@ -174,6 +174,7 @@ Milky Way ; dropAgainWithWildcardAfterEval +required_capability:drop_again_with_wildcard_after_eval from languages | eval language_code = 12, x = 13 | drop language_code diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 7dd679e681798..ad0ef458a5686 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1017,7 +1017,13 @@ public enum Cap { * During resolution (pre-analysis) we have to consider that joins or enriches can override EVALuated values * https://github.com/elastic/elasticsearch/issues/126419 */ - FIX_JOIN_MASKING_EVAL; + FIX_JOIN_MASKING_EVAL, + + /** + * Support for keeping `DROP` attributes when resolving field names. + * see ES|QL: no matches for pattern #126418 + */ + DROP_AGAIN_WITH_WILDCARD_AFTER_EVAL; private final boolean enabled; From 754daf31c9e18700820859c7a576fe5ccf0d5250 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 18 Apr 2025 09:53:00 +0800 Subject: [PATCH 6/9] Reformat code --- .../esql/qa/testFixtures/src/main/resources/drop.csv-spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec index 51597abb062d4..eec8e073e3eec 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec @@ -174,7 +174,7 @@ Milky Way ; dropAgainWithWildcardAfterEval -required_capability:drop_again_with_wildcard_after_eval +required_capability: drop_again_with_wildcard_after_eval from languages | eval language_code = 12, x = 13 | drop language_code From 16e50d09fb143b60084a6cf676a3db90c9b345f4 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Wed, 23 Apr 2025 10:37:06 +0800 Subject: [PATCH 7/9] Update tests --- .../src/main/resources/lookup-join.csv-spec | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index b9c65707146ff..9aaf5552c7aed 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1650,3 +1650,26 @@ event_duration:long 2764889 3450233 ; + +dropAgainWithWildcardAfterEval2 +required_capability: join_lookup_v12 +required_capability: drop_again_with_wildcard_after_eval +from addresses,cartesian_multipolygons,hosts +| rename city.name as message +| lookup join message_types_lookup on message +| eval card = -6013949614291505456, hOntTwnVC = null, PQAF = null, DXkxCFXyw = null, number = -7336429038807752405 +| eval dewAwHC = -1186293612, message = null +| sort number ASC, street ASC, ip0 DESC, name ASC NULLS FIRST, host ASC +| drop number, host_group, *umber, `city.country.continent.name`, dewAwHC, `zip_code`, `message`, city.country.continent.planet.name, `name`, `ip1`, message, zip_code +| drop description, *e, id +| keep `hOntTwnVC`, city.country.continent.planet.galaxy, street +| limit 5 +; + +hOntTwnVC:null | city.country.continent.planet.galaxy:keyword | street:keyword +null | Milky Way | Kearny St +null | Milky Way | Keizersgracht +null | Milky Way | Marunouchi +null | null | null +null | null | null +; From af9c70d00efa60b20ea28cb7766d5a88d5343642 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Fri, 9 May 2025 10:29:15 +0300 Subject: [PATCH 8/9] Slightly adjust the solution and add a couple of tests --- .../xpack/esql/session/EsqlSession.java | 22 +++++++++------ .../session/IndexResolverFieldNamesTests.java | 28 +++++++++++++++++++ 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index f24363cc4a968..91c8547dc1bfb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -590,9 +590,15 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy } var referencesBuilder = AttributeSet.builder(); - // "keep" attributes are special whenever a wildcard is used in their name + // "keep" and "drop" attributes are special whenever a wildcard is used in their name, as the wildcard can shadow some + // attributes ("lookup join" generated columns among others) and steps like removal of Aliases should ignore the fields + // to remove if their name matches one of these wildcards. + // // ie "from test | eval lang = languages + 1 | keep *l" should consider both "languages" and "*l" as valid fields to ask for - var keepCommandRefsBuilder = AttributeSet.builder(); + // "from test | eval first_name = 1 | drop first_name | drop *name should also consider "*name" as valid field to ask for + // + // NOTE: the grammar allows wildcards to be used in other commands as well, but these are forbidden in the LogicalPlanBuilder + var shadowingRefsBuilder = AttributeSet.builder(); var keepJoinRefsBuilder = AttributeSet.builder(); Set wildcardJoinIndices = new java.util.HashSet<>(); @@ -617,12 +623,12 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy if (join.config().type() instanceof JoinTypes.UsingJoinType usingJoinType) { keepJoinRefsBuilder.addAll(usingJoinType.columns()); } - if (keepCommandRefsBuilder.isEmpty()) { + if (shadowingRefsBuilder.isEmpty()) { // No KEEP commands after the JOIN, so we need to mark this index for "*" field resolution wildcardJoinIndices.add(((UnresolvedRelation) join.right()).indexPattern().indexPattern()); } else { // Keep commands can reference the join columns with names that shadow aliases, so we block their removal - keepJoinRefsBuilder.addAll(keepCommandRefsBuilder); + keepJoinRefsBuilder.addAll(shadowingRefsBuilder); } } else { referencesBuilder.addAll(p.references()); @@ -634,12 +640,10 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy p.forEachExpression(UnresolvedNamePattern.class, up -> { var ua = new UnresolvedAttribute(up.source(), up.name()); referencesBuilder.add(ua); - if (p instanceof Keep || p instanceof Drop) { - keepCommandRefsBuilder.add(ua); - } + shadowingRefsBuilder.add(ua); }); if (p instanceof Keep) { - keepCommandRefsBuilder.addAll(p.references()); + shadowingRefsBuilder.addAll(p.references()); } } @@ -669,7 +673,7 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy if (fieldNames.contains(alias.name())) { return; } - referencesBuilder.removeIf(attr -> matchByName(attr, alias.name(), keepCommandRefsBuilder.contains(attr))); + referencesBuilder.removeIf(attr -> matchByName(attr, alias.name(), shadowingRefsBuilder.contains(attr))); }); } }); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 275bfcb011969..152726bb3cb68 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1720,6 +1720,34 @@ public void testDropAgainWithWildcardAfterEval() { """, Set.of("emp_no", "emp_no.*", "*name", "*name.*")); } + public void testDropWildcardedFields_AfterRename() { + assertFieldNames( + """ + from employees + | rename first_name AS first_names, last_name AS last_names + | eval first_names = 1 + | drop first_names + | drop *_names + | keep gender""", + Set.of("first_name", "first_name.*", "last_name", "last_name.*", "*_names", "*_names.*", "gender", "gender.*") + ); + } + + public void testDropWildcardFields_WithLookupJoin() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | LOOKUP JOIN clientips_lookup ON client_ip + | LOOKUP JOIN message_types_lookup ON message + | KEEP @timestamp, message, *e* + | SORT @timestamp + | DROP *e""", + Set.of("client_ip", "client_ip.*", "message", "message.*", "@timestamp", "@timestamp.*", "*e*", "*e", "*e.*"), + Set.of() + ); + } + private Set fieldNames(String query, Set enrichPolicyMatchFields) { var preAnalysisResult = new EsqlSession.PreAnalysisResult(null); return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames(); From 6d80d58588b30fbee4c5510a69927a1418cf9c7f Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Fri, 9 May 2025 11:22:30 +0300 Subject: [PATCH 9/9] Add capability to test --- .../xpack/esql/session/IndexResolverFieldNamesTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 152726bb3cb68..539036d7d1cf0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1734,6 +1734,7 @@ public void testDropWildcardedFields_AfterRename() { } public void testDropWildcardFields_WithLookupJoin() { + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data