Skip to content

Commit 1560c79

Browse files
astefanastefan
authored
ESQL: Fix union types lost attributes in StubRelation for inlinestats (#135547)
This accounts for synthetics attributes that are added as part of union types support inside `StubRelation` (an essential component of `inline stats` support). StubRelation can be regarded as a source plan node (similar to EsRelation) and, thus, the synthetics attributes handling as part of union types feature should be mirrored in this "source" plan as well. ~To make this handling "encapsulated" and tightly linked to StubRelation, the constructor of the class has been changed so that the `output` list of Attributes is computed only in one place and be "reused" whenever a StubRelation is created or changed.~
1 parent a2279e6 commit 1560c79

File tree

6 files changed

+108
-53
lines changed

6 files changed

+108
-53
lines changed

‎docs/changelog/135547.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135547
2+
summary: Fix union types lost attributes in `StubRelation` for inlinestats
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec‎

Lines changed: 83 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -3498,86 +3498,121 @@ warningRegex:java.lang.IllegalArgumentException: milliSeconds \[-1457696696640\]
34983498
2023-10-23T13:53:55.832Z |2023-10-23T13:53:55.832987654Z|1698069235832987654|19
34993499
;
35003500

3501-
ImplicitCastingMultiTypedDateTruncInlinestats_By-Ignore
3502-
required_capability: inline_stats
3503-
// https://github.com/elastic/elasticsearch/issues/133973
3504-
// optimized incorrectly due to missing references [$$emp_no$converted_to$long{f$}#
3501+
InlineStatsCountUnionType
3502+
required_capability: inline_stats_with_union_types_in_stub_relation
3503+
3504+
FROM employees, employees_incompatible
3505+
| KEEP emp_no, hire_date
3506+
| INLINE STATS c = count(emp_no::long)
3507+
| SORT hire_date DESC
3508+
| LIMIT 10
3509+
;
3510+
3511+
emp_no:unsupported | hire_date:date_nanos | c:long
3512+
null | 1999-04-30T00:00:00.000Z | 200
3513+
null | 1999-04-30T00:00:00.000Z | 200
3514+
null | 1997-05-19T00:00:00.000Z | 200
3515+
null | 1997-05-19T00:00:00.000Z | 200
3516+
null | 1996-11-05T00:00:00.000Z | 200
3517+
null | 1996-11-05T00:00:00.000Z | 200
3518+
null | 1995-12-15T00:00:00.000Z | 200
3519+
null | 1995-12-15T00:00:00.000Z | 200
3520+
null | 1995-08-22T00:00:00.000Z | 200
3521+
null | 1995-08-22T00:00:00.000Z | 200
3522+
;
3523+
3524+
ImplicitCastingMultiTypedDateTruncInlinestats_By
3525+
required_capability: inline_stats_with_union_types_in_stub_relation
35053526

35063527
FROM employees, employees_incompatible
35073528
| KEEP emp_no, hire_date
35083529
| INLINE STATS c = count(emp_no::long) BY yr = DATE_TRUNC(1 year, hire_date)
3509-
| SORT yr DESC
3510-
| LIMIT 5
3530+
| SORT yr DESC, hire_date DESC
3531+
| LIMIT 10
35113532
;
35123533

3513-
c:long | yr:date_nanos
3514-
2 | 1999-01-01T00:00:00.000Z
3515-
2 | 1997-01-01T00:00:00.000Z
3516-
2 | 1996-01-01T00:00:00.000Z
3517-
10 | 1995-01-01T00:00:00.000Z
3518-
8 | 1994-01-01T00:00:00.000Z
3534+
emp_no:unsupported | hire_date:date_nanos |c:long | yr:date_nanos
3535+
null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z
3536+
null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z
3537+
null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z
3538+
null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z
3539+
null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z
3540+
null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z
3541+
null |1995-12-15T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
3542+
null |1995-12-15T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
3543+
null |1995-08-22T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
3544+
null |1995-08-22T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
35193545
;
35203546

3521-
ImplicitCastingMultiTypedDateTruncInlinestats_ByWithFilter-Ignore
3522-
required_capability: inline_stats
3523-
// https://github.com/elastic/elasticsearch/issues/133973
3524-
// optimized incorrectly due to missing references [$$emp_no$converted_to$long{f$}#
3547+
ImplicitCastingMultiTypedDateTruncInlinestats_ByWithFilter
3548+
required_capability: inline_stats_with_union_types_in_stub_relation
35253549

35263550
FROM employees, employees_incompatible
35273551
| KEEP emp_no, hire_date
35283552
| INLINE STATS c = count(emp_no::long) where hire_date > "1996-01-01" BY yr = DATE_TRUNC(1 year, hire_date)
3529-
| SORT yr DESC
3530-
| LIMIT 5
3553+
| SORT yr DESC, hire_date DESC
3554+
| LIMIT 10
35313555
;
35323556

3533-
c:long | yr:date_nanos
3534-
2 | 1999-01-01T00:00:00.000Z
3535-
2 | 1997-01-01T00:00:00.000Z
3536-
2 | 1996-01-01T00:00:00.000Z
3537-
0 | 1995-01-01T00:00:00.000Z
3538-
0 | 1994-01-01T00:00:00.000Z
3557+
emp_no:unsupported | hire_date:date_nanos |c:long | yr:date_nanos
3558+
null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z
3559+
null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z
3560+
null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z
3561+
null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z
3562+
null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z
3563+
null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z
3564+
null |1995-12-15T00:00:00.000Z|0 |1995-01-01T00:00:00.000Z
3565+
null |1995-12-15T00:00:00.000Z|0 |1995-01-01T00:00:00.000Z
3566+
null |1995-08-22T00:00:00.000Z|0 |1995-01-01T00:00:00.000Z
3567+
null |1995-08-22T00:00:00.000Z|0 |1995-01-01T00:00:00.000Z
35393568
;
35403569

3541-
ImplicitCastingMultiTypedDateTruncInlinestats_ByWithEval-Ignore
3542-
required_capability: inline_stats
3543-
// https://github.com/elastic/elasticsearch/issues/133973
3544-
// optimized incorrectly due to missing references [$$emp_no$converted_to$long{f$}#
3570+
ImplicitCastingMultiTypedDateTruncInlinestats_ByWithEval
3571+
required_capability: inline_stats_with_union_types_in_stub_relation
35453572

35463573
FROM employees, employees_incompatible
35473574
| KEEP emp_no, hire_date
35483575
| EVAL yr = DATE_TRUNC(1 year, hire_date)
35493576
| INLINE STATS c = count(emp_no::long) BY yr
3550-
| SORT yr DESC
3551-
| LIMIT 5
3577+
| SORT yr DESC, hire_date DESC
3578+
| LIMIT 10
35523579
;
35533580

3554-
c:long | yr:date_nanos
3555-
2 | 1999-01-01T00:00:00.000Z
3556-
2 | 1997-01-01T00:00:00.000Z
3557-
2 | 1996-01-01T00:00:00.000Z
3558-
10 | 1995-01-01T00:00:00.000Z
3559-
8 | 1994-01-01T00:00:00.000Z
3581+
emp_no:unsupported | hire_date:date_nanos |c:long | yr:date_nanos
3582+
null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z
3583+
null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z
3584+
null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z
3585+
null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z
3586+
null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z
3587+
null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z
3588+
null |1995-12-15T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
3589+
null |1995-12-15T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
3590+
null |1995-08-22T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
3591+
null |1995-08-22T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
35603592
;
35613593

3562-
ImplicitCastingMultiTypedDateTruncInlinestats_ByWithEvalWithFilter-Ignore
3563-
required_capability: inline_stats
3564-
// https://github.com/elastic/elasticsearch/issues/133973
3565-
// optimized incorrectly due to missing references [$$emp_no$converted_to$long{f$}#
3594+
ImplicitCastingMultiTypedDateTruncInlinestats_ByWithEvalWithFilter
3595+
required_capability: inline_stats_with_union_types_in_stub_relation
35663596

35673597
FROM employees, employees_incompatible
35683598
| KEEP emp_no, hire_date
35693599
| EVAL yr = DATE_TRUNC(1 year, hire_date)
35703600
| INLINE STATS c = count(emp_no::long) where hire_date > "1991-01-01" BY yr
3571-
| SORT yr DESC
3572-
| LIMIT 5
3601+
| SORT yr DESC, hire_date DESC
3602+
| LIMIT 10
35733603
;
35743604

3575-
c:long | yr:date_nanos
3576-
2 | 1999-01-01T00:00:00.000Z
3577-
2 | 1997-01-01T00:00:00.000Z
3578-
2 | 1996-01-01T00:00:00.000Z
3579-
10 | 1995-01-01T00:00:00.000Z
3580-
8 | 1994-01-01T00:00:00.000Z
3605+
emp_no:unsupported | hire_date:date_nanos |c:long | yr:date_nanos
3606+
null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z
3607+
null |1999-04-30T00:00:00.000Z|2 |1999-01-01T00:00:00.000Z
3608+
null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z
3609+
null |1997-05-19T00:00:00.000Z|2 |1997-01-01T00:00:00.000Z
3610+
null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z
3611+
null |1996-11-05T00:00:00.000Z|2 |1996-01-01T00:00:00.000Z
3612+
null |1995-12-15T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
3613+
null |1995-12-15T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
3614+
null |1995-08-22T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
3615+
null |1995-08-22T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
35813616
;
35823617

35833618
ImplicitCastingMultiTypedBucketDateNanosByYear

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1543,6 +1543,8 @@ public enum Cap {
15431543
/** INLINE STATS supports remote indices */
15441544
INLINE_STATS_SUPPORTS_REMOTE(INLINESTATS_V11.enabled),
15451545

1546+
INLINE_STATS_WITH_UNION_TYPES_IN_STUB_RELATION(INLINE_STATS.enabled),
1547+
15461548
/**
15471549
* Support TS command in non-snapshot builds
15481550
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvals.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
import java.util.List;
2222
import java.util.Map;
2323

24+
import static org.elasticsearch.xpack.esql.plan.logical.join.InlineJoin.replaceStub;
25+
import static org.elasticsearch.xpack.esql.plan.logical.join.StubRelation.computeOutput;
26+
2427
/**
2528
* Replace any evaluation from the inlined aggregation side (right side) to the left side (source) to perform the matching.
2629
* In INLINE STATS m = MIN(x) BY a + b the right side contains STATS m = MIN(X) BY a + b.
@@ -83,8 +86,7 @@ protected LogicalPlan rule(InlineJoin plan) {
8386
if (groupingAlias.size() > 0) {
8487
left = new Eval(plan.source(), plan.left(), groupingAlias);
8588
}
86-
8789
// replace the old stub with the new out to capture the new output
88-
return plan.replaceChildren(left, InlineJoin.replaceStub(new StubRelation(right.source(), left.output()), right));
90+
return plan.replaceChildren(left, replaceStub(new StubRelation(right.source(), computeOutput(right, left)), right));
8991
}
9092
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public class InlineJoin extends Join {
5555
* Replaces the source of the target plan with a stub preserving the output of the source plan.
5656
*/
5757
public static LogicalPlan stubSource(UnaryPlan sourcePlan, LogicalPlan target) {
58-
return sourcePlan.replaceChild(new StubRelation(sourcePlan.source(), target.output()));
58+
return sourcePlan.replaceChild(new StubRelation(sourcePlan.source(), StubRelation.computeOutput(sourcePlan, target)));
5959
}
6060

6161
/**

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/StubRelation.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@
1818
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
1919

2020
import java.io.IOException;
21+
import java.util.ArrayList;
22+
import java.util.LinkedHashSet;
2123
import java.util.List;
2224
import java.util.Objects;
25+
import java.util.Set;
2326

2427
import static java.util.Collections.emptyList;
2528

@@ -36,15 +39,23 @@ public class StubRelation extends LeafPlan {
3639
StubRelation::new
3740
);
3841

39-
public static final StubRelation EMPTY = new StubRelation(null, null);
40-
4142
private final List<Attribute> output;
4243

4344
public StubRelation(Source source, List<Attribute> output) {
4445
super(source);
4546
this.output = output;
4647
}
4748

49+
/*
50+
* The output of a StubRelation must also include any synthetic attributes referenced by the source plan (union types is a great
51+
* example of those attributes that has some special treatment throughout the planning phases, especially in the EsRelation).
52+
*/
53+
public static List<Attribute> computeOutput(LogicalPlan source, LogicalPlan target) {
54+
Set<Attribute> stubRelationOutput = new LinkedHashSet<>(target.output());
55+
stubRelationOutput.addAll(source.references().stream().filter(Attribute::synthetic).toList());
56+
return new ArrayList<>(stubRelationOutput);
57+
}
58+
4859
public StubRelation(StreamInput in) throws IOException {
4960
this(Source.readFrom((PlanStreamInput) in), emptyList());
5061
}

0 commit comments

Comments
 (0)