Skip to content

Commit bb03beb

Browse files
[9.0] ESQL: Fix inconsistent column order in MV_EXPAND (#129745) (#131428)
* ESQL: Fix inconsistent column order in MV_EXPAND (#129745) The new attribute generated by MV_EXPAND should remain in the original position. The projection added by ProjectAwayColumns does not respect the original order of attributes. Make ProjectAwayColumns respect the order of attributes to fix this. (cherry picked from commit ac0c508) # Conflicts: # x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java # x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java # x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java # x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java * Adapt approach for missing AttributeSetBuilder Builder is only available on main/9.x, so we have to correctly use the immutable AttributeSets directly. --------- Co-authored-by: kanoshiou <[email protected]>
1 parent efbff12 commit bb03beb

File tree

6 files changed

+360
-238
lines changed

6 files changed

+360
-238
lines changed

docs/changelog/129745.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 129745
2+
summary: "ESQL: Fix `mv_expand` inconsistent column order"
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 129000

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,5 +234,9 @@ public boolean isEmpty() {
234234
public AttributeSet build() {
235235
return new AttributeSet(mapBuilder.build());
236236
}
237+
238+
public void clear() {
239+
mapBuilder.keySet().clear();
240+
}
237241
}
238242
}

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,3 +418,65 @@ emp_no:integer | job_positions:keyword
418418
10001 | Accountant
419419
10001 | Senior Python Developer
420420
;
421+
422+
testMvExpandInconsistentColumnOrder1
423+
required_capability: fix_mv_expand_inconsistent_column_order
424+
from message_types
425+
| eval foo_1 = 1, foo_2 = 2
426+
| sort message
427+
| mv_expand foo_1
428+
;
429+
430+
message:keyword | type:keyword | foo_1:integer | foo_2:integer
431+
Connected to 10.1.0.1 | Success | 1 | 2
432+
Connected to 10.1.0.2 | Success | 1 | 2
433+
Connected to 10.1.0.3 | Success | 1 | 2
434+
Connection error | Error | 1 | 2
435+
Development environment | Development | 1 | 2
436+
Disconnected | Disconnected | 1 | 2
437+
Production environment | Production | 1 | 2
438+
;
439+
440+
testMvExpandInconsistentColumnOrder2
441+
required_capability: fix_mv_expand_inconsistent_column_order
442+
from message_types
443+
| eval foo_1 = [1, 3], foo_2 = 2
444+
| sort message
445+
| mv_expand foo_1
446+
;
447+
448+
message:keyword | type:keyword | foo_1:integer | foo_2:integer
449+
Connected to 10.1.0.1 | Success | 1 | 2
450+
Connected to 10.1.0.1 | Success | 3 | 2
451+
Connected to 10.1.0.2 | Success | 1 | 2
452+
Connected to 10.1.0.2 | Success | 3 | 2
453+
Connected to 10.1.0.3 | Success | 1 | 2
454+
Connected to 10.1.0.3 | Success | 3 | 2
455+
Connection error | Error | 1 | 2
456+
Connection error | Error | 3 | 2
457+
Development environment | Development | 1 | 2
458+
Development environment | Development | 3 | 2
459+
Disconnected | Disconnected | 1 | 2
460+
Disconnected | Disconnected | 3 | 2
461+
Production environment | Production | 1 | 2
462+
Production environment | Production | 3 | 2
463+
;
464+
465+
testMvExpandInconsistentColumnOrder3
466+
required_capability: fix_mv_expand_inconsistent_column_order
467+
from message_types
468+
| sort type
469+
| eval language_code = 1, `language_name` = false, message = true, foo_3 = 1, foo_2 = null
470+
| eval foo_3 = "1", `foo_3` = -1, foo_1 = 1, `language_code` = null, `foo_2` = "1"
471+
| mv_expand foo_1
472+
| limit 5
473+
;
474+
475+
type:keyword | language_name:boolean | message:boolean | foo_3:integer | foo_1:integer | language_code:null | foo_2:keyword
476+
Development | false | true | -1 | 1 | null | 1
477+
Disconnected | false | true | -1 | 1 | null | 1
478+
Error | false | true | -1 | 1 | null | 1
479+
Production | false | true | -1 | 1 | null | 1
480+
Success | false | true | -1 | 1 | null | 1
481+
;
482+

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -930,7 +930,13 @@ public enum Cap {
930930
/**
931931
* Guards a bug fix matching {@code TO_LOWER(f) == ""}.
932932
*/
933-
TO_LOWER_EMPTY_STRING;
933+
TO_LOWER_EMPTY_STRING,
934+
935+
/**
936+
* Support for the mv_expand target attribute should be retained in its original position.
937+
* see <a href="https://github.com/elastic/elasticsearch/issues/129000"> ES|QL: inconsistent column order #129000 </a>
938+
*/
939+
FIX_MV_EXPAND_INCONSISTENT_COLUMN_ORDER;
934940

935941
private final boolean enabled;
936942

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,22 @@ public PhysicalPlan apply(PhysicalPlan plan) {
5757

5858
// no need for projection when dealing with aggs
5959
if (logicalFragment instanceof Aggregate == false) {
60-
List<Attribute> output = new ArrayList<>(requiredAttributes.get());
60+
// we should respect the order of the attributes
61+
List<Attribute> output = new ArrayList<>();
62+
for (Attribute attribute : logicalFragment.output()) {
63+
if (requiredAttributes.get().contains(attribute)) {
64+
output.add(attribute);
65+
}
66+
}
67+
// requiredAttributes should only have attributes that are also in the fragment's output.
68+
// This assumption can be wrong in case of remote ENRICH, see https://github.com/elastic/elasticsearch/issues/118531
69+
// TODO: stop adding the remaining required attributes once remote ENRICH is fixed.
70+
if (output.size() != requiredAttributes.get().size()) {
71+
AttributeSet alreadyAdded = AttributeSet.of(output);
72+
AttributeSet remaining = requiredAttributes.get().subtract(alreadyAdded);
73+
output.addAll(remaining);
74+
}
75+
6176
// if all the fields are filtered out, it's only the count that matters
6277
// however until a proper fix (see https://github.com/elastic/elasticsearch/issues/98703)
6378
// add a synthetic field (so it doesn't clash with the user defined one) to return a constant

0 commit comments

Comments
 (0)