From 5b9070c3892b8c056a698872ba7947627e2435ae Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Tue, 11 Feb 2025 21:10:22 +0800 Subject: [PATCH 1/8] Swapping names --- .../org/elasticsearch/xpack/esql/analysis/Analyzer.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 1351b5ce51f44..1072d6a6ccebe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -848,11 +848,18 @@ public static List projectionsForRename(Rename rename, List unresolved = new ArrayList<>(renamingsCount); Map reverseAliasing = new HashMap<>(renamingsCount); // `| rename a as x` => map(a: x) + Set toRenameAttributes = rename.renamings() + .stream() + .map(Alias::child) + .filter(v -> v instanceof UnresolvedAttribute) + .map(attr -> ((UnresolvedAttribute) attr).name()) + .collect(Collectors.toSet()); + rename.renamings().forEach(alias -> { // skip NOPs: `| rename a as a` if (alias.child() instanceof UnresolvedAttribute ua && alias.name().equals(ua.name()) == false) { // remove attributes overwritten by a renaming: `| keep a, b, c | rename a as b` - projections.removeIf(x -> x.name().equals(alias.name())); + projections.removeIf(x -> x.name().equals(alias.name()) && toRenameAttributes.contains(x.name()) == false); var resolved = maybeResolveAttribute(ua, childrenOutput, logger); if (resolved instanceof UnsupportedAttribute || resolved.resolved()) { From 413c885a394bd7e7156bd678e00de7a152ff2b82 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Tue, 11 Feb 2025 21:29:57 +0800 Subject: [PATCH 2/8] Add tests --- .../qa/testFixtures/src/main/resources/rename.csv-spec | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec index ca4c627cae749..c84a20e940ba7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec @@ -213,3 +213,12 @@ x:keyword Facello Simmel ; + +swappingName +row foo = 10, bar = null +| rename bar AS foo, foo AS bar +; + +bar:INTEGER | foo:NULL +10 | null +; From ec26025a1b8d6eefeea5c19d452b23ae9f79e0e2 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Tue, 11 Feb 2025 21:30:24 +0800 Subject: [PATCH 3/8] Add changelog --- docs/changelog/122250.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/122250.yaml diff --git a/docs/changelog/122250.yaml b/docs/changelog/122250.yaml new file mode 100644 index 0000000000000..7815c6604e2a5 --- /dev/null +++ b/docs/changelog/122250.yaml @@ -0,0 +1,6 @@ +pr: 122250 +summary: "ES|QL: Improve `RENAME` logic to swap names" +area: ES|QL +type: enhancement +issues: + - 121739 From ce12e7bdbfe38ed7e802c6efa81fa7109927fb01 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Thu, 13 Feb 2025 17:24:29 +0800 Subject: [PATCH 4/8] Update testcase --- .../src/main/resources/rename.csv-spec | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec index c84a20e940ba7..c14c407f352fa 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec @@ -214,11 +214,15 @@ Facello Simmel ; -swappingName -row foo = 10, bar = null -| rename bar AS foo, foo AS bar +swappingNames +FROM employees +| SORT emp_no ASC +| KEEP first_name, last_name +| RENAME first_name AS last_name, last_name AS first_name +| LIMIT 2 ; -bar:INTEGER | foo:NULL -10 | null +last_name:keyword | first_name:keyword +Georgi | Facello +Bezalel | Simmel ; From 619373a4bb234e6e6012f063560673bf0c561408 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 14 Feb 2025 14:51:34 +0800 Subject: [PATCH 5/8] Add more tests --- .../src/main/resources/rename.csv-spec | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec index c14c407f352fa..22cacf8291568 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec @@ -218,11 +218,24 @@ swappingNames FROM employees | SORT emp_no ASC | KEEP first_name, last_name -| RENAME first_name AS last_name, last_name AS first_name +| RENAME first_name AS last_name, last_name AS first_name, first_name as name | LIMIT 2 ; -last_name:keyword | first_name:keyword +last_name:keyword | name:keyword Georgi | Facello Bezalel | Simmel ; + +complexSwappingNames +FROM employees +| SORT emp_no ASC +| KEEP first_name, last_name, emp_no +| RENAME first_name AS last_name, last_name AS first_name, first_name as emp_no, emp_no AS first_name +| LIMIT 2 +; + +last_name:keyword | emp_no:keyword | first_name:INTEGER +Georgi | Facello | 10001 +Bezalel | Simmel | 10002 +; From 353fdd7bf2a13bde2a7ec1a349ae294c13ccaf33 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Mon, 17 Mar 2025 19:58:52 +0800 Subject: [PATCH 6/8] Update --- .../src/main/resources/rename.csv-spec | 62 +++++++++++++++++-- .../xpack/esql/analysis/Analyzer.java | 10 +-- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec index e13ce0975d10e..f3a101f681c59 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec @@ -223,9 +223,9 @@ FROM employees | LIMIT 2 ; -last_name:keyword | name:keyword -Georgi | Facello -Bezalel | Simmel +name:keyword +Georgi +Bezalel ; complexSwappingNames @@ -236,7 +236,57 @@ FROM employees | LIMIT 2 ; -last_name:keyword | emp_no:keyword | first_name:INTEGER -Georgi | Facello | 10001 -Bezalel | Simmel | 10002 +first_name:keyword +Georgi +Bezalel +; + + +reuseRenamedAlias +FROM employees +| SORT emp_no ASC +| KEEP first_name, last_name +| LIMIT 2 +| RENAME first_name AS x, x AS y, last_name as x +; + +y:keyword | x:keyword +Georgi | Facello +Bezalel | Simmel +; + + +multipleRenamesToSameAliasLastOnePrevails +FROM employees +| SORT emp_no ASC +| KEEP first_name, last_name +| LIMIT 2 +| RENAME first_name AS x, last_name as x +; + +x:keyword +Facello +Simmel +; + + +swapNames +ROW a="keyword", b=5 +| RENAME a AS temp, b AS a, temp AS b +; + +b:keyword | a:integer +keyword | 5 +; + + +multipleRenames +ROW a="keyword", b=5, c=null +| RENAME a AS c, b AS a +| RENAME c AS b +| RENAME a AS b, b AS a +; + +a:integer +5 ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 4cd288eb9031a..fc04fa2b1636c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -981,18 +981,12 @@ public static List projectionsForRename(Rename rename, List unresolved = new ArrayList<>(renamingsCount); Map reverseAliasing = new HashMap<>(renamingsCount); // `| rename a as x` => map(a: x) - Set toRenameAttributes = rename.renamings() - .stream() - .map(Alias::child) - .filter(v -> v instanceof UnresolvedAttribute) - .map(attr -> ((UnresolvedAttribute) attr).name()) - .collect(Collectors.toSet()); - rename.renamings().forEach(alias -> { // skip NOPs: `| rename a as a` if (alias.child() instanceof UnresolvedAttribute ua && alias.name().equals(ua.name()) == false) { // remove attributes overwritten by a renaming: `| keep a, b, c | rename a as b` - projections.removeIf(x -> x.name().equals(alias.name()) && toRenameAttributes.contains(x.name()) == false); + projections.removeIf(x -> x.name().equals(alias.name())); + childrenOutput.removeIf(x -> x.name().equals(alias.name())); var resolved = maybeResolveAttribute(ua, childrenOutput, logger); if (resolved instanceof UnsupportedAttribute || resolved.resolved()) { From 445c4fb505bb387fec1bcb33440facf6dc62cff6 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Mon, 17 Mar 2025 20:18:03 +0800 Subject: [PATCH 7/8] Update changelog --- docs/changelog/122250.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/122250.yaml b/docs/changelog/122250.yaml index 7815c6604e2a5..3888e9dc179cb 100644 --- a/docs/changelog/122250.yaml +++ b/docs/changelog/122250.yaml @@ -1,5 +1,5 @@ pr: 122250 -summary: "ES|QL: Improve `RENAME` logic to swap names" +summary: "ESQL: Align `RENAME` behavior with `EVAL` for sequential processing" area: ES|QL type: enhancement issues: From 8d84d3d2d8ad3b994d4c410ebf60a3de8054bb01 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Tue, 18 Mar 2025 09:32:19 +0800 Subject: [PATCH 8/8] Fix bwc --- .../esql/qa/testFixtures/src/main/resources/rename.csv-spec | 6 ++++++ .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec index f3a101f681c59..71603585dc64e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec @@ -216,6 +216,7 @@ Simmel ; swappingNames +required_capability: rename_sequential_processing FROM employees | SORT emp_no ASC | KEEP first_name, last_name @@ -229,6 +230,7 @@ Bezalel ; complexSwappingNames +required_capability: rename_sequential_processing FROM employees | SORT emp_no ASC | KEEP first_name, last_name, emp_no @@ -243,6 +245,7 @@ Bezalel reuseRenamedAlias +required_capability: rename_sequential_processing FROM employees | SORT emp_no ASC | KEEP first_name, last_name @@ -257,6 +260,7 @@ Bezalel | Simmel multipleRenamesToSameAliasLastOnePrevails +required_capability: rename_sequential_processing FROM employees | SORT emp_no ASC | KEEP first_name, last_name @@ -271,6 +275,7 @@ Simmel swapNames +required_capability: rename_sequential_processing ROW a="keyword", b=5 | RENAME a AS temp, b AS a, temp AS b ; @@ -281,6 +286,7 @@ keyword | 5 multipleRenames +required_capability: rename_sequential_processing ROW a="keyword", b=5, c=null | RENAME a AS c, b AS a | RENAME c AS b diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index ca3ceafaf8bb8..833d8fa12a760 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -407,6 +407,12 @@ public enum Cap { */ UNION_TYPES_FIX_RENAME_RESOLUTION, + /** + * Execute `RENAME` operations sequentially from left to right, + * see ESQL: Align RENAME behavior with EVAL for sequential processing #122250 + */ + RENAME_SEQUENTIAL_PROCESSING, + /** * Fix for union-types when some indexes are missing the required field. Done in #111932. */