Skip to content

Commit 46d25e6

Browse files
prwhelanprwhelan
authored
Merge branch '9.3' into backport/9.3/pr-139765
2 parents e6627dd + b2fcef5 commit 46d25e6

File tree

10 files changed

+119
-96
lines changed

10 files changed

+119
-96
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-rate.csv-spec‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,3 +335,62 @@ rate_bytes_in:double | cluster:keyword | time_bucket:datetime
335335
8.939832 | qa | 2024-05-10T00:14:00.000Z
336336
9.2257 | prod | 2024-05-10T00:20:00.000Z
337337
;
338+
339+
bare_rate_outputs_dimensions
340+
required_capability: ts_command_v0
341+
required_capability: metrics_group_by_all
342+
required_capability: metrics_group_by_all_with_ts_dimensions
343+
344+
TS k8s
345+
| STATS rate=rate(network.total_bytes_in)
346+
| SORT rate DESC
347+
;
348+
ignoreOrder:true
349+
350+
rate:double | _timeseries:keyword
351+
4.689830508474576 | "{""cluster"":""staging"",""pod"":""one"",""region"":""us""}"
352+
6.54326561324304 | "{""cluster"":""staging"",""pod"":""three"",""region"":""us""}"
353+
7.340495867768595 | "{""cluster"":""prod"",""pod"":""two"",""region"":[""eu"",""us""]}"
354+
7.377611940298507 | "{""cluster"":""staging"",""pod"":""two"",""region"":""us""}"
355+
8.434290687554395 | "{""cluster"":""prod"",""pod"":""three"",""region"":[""eu"",""us""]}"
356+
8.716707021791768 | "{""cluster"":""prod"",""pod"":""one"",""region"":[""eu"",""us""]}"
357+
9.485643970467596 | "{""cluster"":""qa"",""pod"":""three""}"
358+
10.649149922720246 | "{""cluster"":""qa"",""pod"":""one""}"
359+
13.17372515125324 | "{""cluster"":""qa"",""pod"":""two""}"
360+
;
361+
362+
bare_rate_with_tbucket_outputs_dimensions
363+
required_capability: ts_command_v0
364+
required_capability: metrics_group_by_all
365+
required_capability: metrics_group_by_all_with_ts_dimensions
366+
367+
TS k8s
368+
| STATS rate=rate(network.total_bytes_in) BY tbucket = TBUCKET(1 hour)
369+
| SORT rate DESC
370+
;
371+
ignoreOrder:true
372+
373+
rate:double | _timeseries:keyword | tbucket:datetime
374+
1.6554700854700857 | "{""cluster"":""staging"",""pod"":""one"",""region"":""us""}" | 2024-05-10T00:00:00.000Z
375+
2.5258595644176904 | "{""cluster"":""staging"",""pod"":""three"",""region"":""us""}" | 2024-05-10T00:00:00.000Z
376+
2.621423611111111 | "{""cluster"":""prod"",""pod"":""two"",""region"":[""eu"",""us""]}" | 2024-05-10T00:00:00.000Z
377+
2.830347222222222 | "{""cluster"":""prod"",""pod"":""three"",""region"":[""eu"",""us""]}" | 2024-05-10T00:00:00.000Z
378+
2.874194651741293 | "{""cluster"":""staging"",""pod"":""two"",""region"":""us""}" | 2024-05-10T00:00:00.000Z
379+
3.1304347826086953 | "{""cluster"":""prod"",""pod"":""one"",""region"":[""eu"",""us""]}" | 2024-05-10T00:00:00.000Z
380+
3.3457754629629632 | "{""cluster"":""qa"",""pod"":""three""}" | 2024-05-10T00:00:00.000Z
381+
3.959770114942529 | "{""cluster"":""qa"",""pod"":""one""}" | 2024-05-10T00:00:00.000Z
382+
4.379885057471264 | "{""cluster"":""qa"",""pod"":""two""}" | 2024-05-10T00:00:00.000Z
383+
;
384+
385+
wrapped_rate
386+
required_capability: ts_command_v0
387+
required_capability: metrics_group_by_all
388+
required_capability: metrics_group_by_all_with_ts_dimensions
389+
390+
TS k8s
391+
| STATS max_rate = MAX(rate(network.total_bytes_in))
392+
| EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate;
393+
394+
max_rate:double
395+
13.173725
396+
;

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec‎

Lines changed: 0 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -901,68 +901,6 @@ sum:double | _timeseries:keyword
901901
210.625 | "{""cluster"":""qa"",""pod"":""one""}" | 2024-05-10T00:00:00.000Z
902902
;
903903

904-
bare_rate_outputs_dimensions
905-
required_capability: ts_command_v0
906-
required_capability: metrics_group_by_all
907-
required_capability: metrics_group_by_all_with_ts_dimensions
908-
909-
910-
TS k8s
911-
| STATS rate=rate(network.total_bytes_in)
912-
| SORT rate DESC
913-
;
914-
ignoreOrder:true
915-
916-
rate:double | _timeseries:keyword
917-
4.689830508474576 | "{""cluster"":""staging"",""pod"":""one"",""region"":""us""}"
918-
6.54326561324304 | "{""cluster"":""staging"",""pod"":""three"",""region"":""us""}"
919-
7.340495867768595 | "{""cluster"":""prod"",""pod"":""two"",""region"":[""eu"",""us""]}"
920-
7.377611940298507 | "{""cluster"":""staging"",""pod"":""two"",""region"":""us""}"
921-
8.434290687554395 | "{""cluster"":""prod"",""pod"":""three"",""region"":[""eu"",""us""]}"
922-
8.716707021791768 | "{""cluster"":""prod"",""pod"":""one"",""region"":[""eu"",""us""]}"
923-
9.485643970467596 | "{""cluster"":""qa"",""pod"":""three""}"
924-
10.649149922720246 | "{""cluster"":""qa"",""pod"":""one""}"
925-
13.17372515125324 | "{""cluster"":""qa"",""pod"":""two""}"
926-
927-
;
928-
929-
bare_rate_with_tbucket_outputs_dimensions
930-
required_capability: ts_command_v0
931-
required_capability: metrics_group_by_all
932-
required_capability: metrics_group_by_all_with_ts_dimensions
933-
934-
TS k8s
935-
| STATS rate=rate(network.total_bytes_in) BY tbucket = TBUCKET(1 hour)
936-
| SORT rate DESC
937-
;
938-
ignoreOrder:true
939-
940-
rate:double | _timeseries:keyword | tbucket:datetime
941-
1.6554700854700857 | "{""cluster"":""staging"",""pod"":""one"",""region"":""us""}" | 2024-05-10T00:00:00.000Z
942-
2.5258595644176904 | "{""cluster"":""staging"",""pod"":""three"",""region"":""us""}" | 2024-05-10T00:00:00.000Z
943-
2.621423611111111 | "{""cluster"":""prod"",""pod"":""two"",""region"":[""eu"",""us""]}" | 2024-05-10T00:00:00.000Z
944-
2.830347222222222 | "{""cluster"":""prod"",""pod"":""three"",""region"":[""eu"",""us""]}" | 2024-05-10T00:00:00.000Z
945-
2.874194651741293 | "{""cluster"":""staging"",""pod"":""two"",""region"":""us""}" | 2024-05-10T00:00:00.000Z
946-
3.1304347826086953 | "{""cluster"":""prod"",""pod"":""one"",""region"":[""eu"",""us""]}" | 2024-05-10T00:00:00.000Z
947-
3.3457754629629632 | "{""cluster"":""qa"",""pod"":""three""}" | 2024-05-10T00:00:00.000Z
948-
3.959770114942529 | "{""cluster"":""qa"",""pod"":""one""}" | 2024-05-10T00:00:00.000Z
949-
4.379885057471264 | "{""cluster"":""qa"",""pod"":""two""}" | 2024-05-10T00:00:00.000Z
950-
951-
;
952-
953-
wrapped_rate
954-
required_capability: ts_command_v0
955-
required_capability: metrics_group_by_all
956-
required_capability: metrics_group_by_all_with_ts_dimensions
957-
958-
TS k8s
959-
| STATS max_rate = MAX(rate(network.total_bytes_in))
960-
| EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate;
961-
962-
max_rate:double
963-
13.173725
964-
;
965-
966904
oneRateIncreaseWithBucketAndClusterThenFilter
967905
required_capability: ts_command_v0
968906
required_capability: rate_with_interpolation

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TSDataGenerationHelper.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,7 @@ private static Object randomDimensionValue(String dimensionName) {
5454
// Making a list-to-set-to-list to ensure uniqueness.
5555
this.numDocs = numDocs;
5656
var maxAttributes = (int) Math.sqrt(numDocs);
57-
List<String> tempAttributeSet = List.copyOf(
58-
Set.copyOf(ESTestCase.randomList(1, maxAttributes, () -> ESTestCase.randomAlphaOfLengthBetween(3, 30)))
59-
);
57+
List<String> tempAttributeSet = ESTestCase.randomList(1, maxAttributes, () -> ESTestCase.randomAlphaOfLengthBetween(3, 30));
6058
var maxTimeSeries = (int) Math.sqrt(numDocs);
6159
var minTimeSeries = Math.max(1, maxTimeSeries / 4);
6260
numTimeSeries = ESTestCase.randomIntBetween(minTimeSeries, maxTimeSeries);

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/TimeSeriesGroupByAll.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,6 @@ public LogicalPlan rule(TimeSeriesAggregate aggregate) {
9090
if (Functions.isGrouping(Alias.unwrap(grouping)) == false) {
9191
throw new IllegalArgumentException(
9292
"Cannot mix time-series aggregate and grouping attributes. Found [" + grouping.sourceText() + "]."
93-
9493
);
9594
}
9695
groupings.add(grouping);

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
1717
import org.elasticsearch.xpack.esql.core.expression.Alias;
1818
import org.elasticsearch.xpack.esql.core.expression.Expression;
19+
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
1920
import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
2021
import org.elasticsearch.xpack.esql.core.expression.function.Function;
2122
import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
@@ -177,9 +178,20 @@ else if (p.resolved()) {
177178
// do groupings first
178179
var groupings = agg.groupings();
179180
groupings.forEach(unresolvedExpressions);
181+
182+
// We don't count _timeseries which is added implicitly to grouping but not to a list of aggs
183+
boolean hasGroupByAll = false;
184+
for (Expression grouping : groupings) {
185+
if (MetadataAttribute.isTimeSeriesAttribute(grouping)) {
186+
hasGroupByAll = true;
187+
break;
188+
}
189+
}
190+
int groupingSize = hasGroupByAll ? groupings.size() - 1 : groupings.size();
191+
180192
// followed by just the aggregates (to avoid going through the groups again)
181193
var aggs = agg.aggregates();
182-
int size = aggs.size() - groupings.size();
194+
int size = aggs.size() - groupingSize;
183195
aggs.subList(0, size).forEach(unresolvedExpressions);
184196
}
185197
// similar approach for Lookup

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,10 @@ public static boolean isScoreAttribute(Expression a) {
170170
return a instanceof MetadataAttribute ma && ma.name().equals(SCORE);
171171
}
172172

173+
public static boolean isTimeSeriesAttribute(Expression a) {
174+
return a instanceof Attribute ma && ma.name().equals(TIMESERIES);
175+
}
176+
173177
@Override
174178
protected int innerHashCode(boolean ignoreIds) {
175179
return Objects.hash(super.innerHashCode(ignoreIds), searchable);

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PostOptimizationPhasePlanVerifier.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,8 @@ private static void verifyOutputNotChanged(QueryPlan<?> optimizedPlan, List<Attr
9090
// TranslateTimeSeriesAggregate may add a _timeseries attribute into the projection
9191
boolean hasTimeSeriesReplacingTsId = optimizedPlan.anyMatch(
9292
a -> a instanceof TimeSeriesAggregate ts
93-
&& ts.output().stream().anyMatch(g -> g.name().equals(MetadataAttribute.TIMESERIES))
94-
&& expectedOutputAttributes.stream().noneMatch(g -> g.name().equals(MetadataAttribute.TIMESERIES))
93+
&& ts.output().stream().anyMatch(MetadataAttribute::isTimeSeriesAttribute)
94+
&& expectedOutputAttributes.stream().noneMatch(MetadataAttribute::isTimeSeriesAttribute)
9595
);
9696

9797
boolean ignoreError = hasProjectAwayColumns || hasLookupJoinExec || hasTextGroupingInTimeSeries || hasTimeSeriesReplacingTsId;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ExtractDimensionFieldsAfterAggregation.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ private PhysicalPlan rule(TimeSeriesAggregateExec oldAgg, LocalPhysicalOptimizer
9393
throw new IllegalStateException("expected one intermediate attribute for [" + af + "] but got [" + size + "]");
9494
}
9595
Attribute oldAttr = oldIntermediates.get(intermediateOffset);
96-
if (dimensionField.name().equals(MetadataAttribute.TIMESERIES)) {
96+
if (MetadataAttribute.isTimeSeriesAttribute(dimensionField)) {
9797
var sourceField = new FieldAttribute(
9898
dimensionField.source(),
9999
null,
@@ -161,7 +161,7 @@ private PhysicalPlan rule(TimeSeriesAggregateExec oldAgg, LocalPhysicalOptimizer
161161

162162
private static Attribute valuesOfDimensionField(AggregateFunction af, AttributeSet inputAttributes) {
163163
if (af instanceof DimensionValues values && values.hasFilter() == false && values.field() instanceof Attribute attr) {
164-
if (inputAttributes.contains(attr) == false || attr.name().equals(MetadataAttribute.TIMESERIES)) {
164+
if (inputAttributes.contains(attr) == false || MetadataAttribute.isTimeSeriesAttribute(attr)) {
165165
return attr;
166166
}
167167
}

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3423,7 +3423,35 @@ public void testTopSnippetsFunctionInvalidInputs() {
34233423
),
34243424
equalTo("1:29: 'num_words' option must be a positive integer, found [0]")
34253425
);
3426+
}
3427+
3428+
public void testTimeSeriesWithUnsupportedDataType() {
3429+
assumeTrue("requires metrics command", EsqlCapabilities.Cap.METRICS_GROUP_BY_ALL_WITH_TS_DIMENSIONS.isEnabled());
3430+
3431+
// GroupByAll
3432+
assertThat(
3433+
error("TS k8s | STATS rate(network.eth0.tx)", k8s),
3434+
equalTo(
3435+
"1:16: first argument of [rate(network.eth0.tx)] must be [counter_long, counter_integer or counter_double], "
3436+
+ "found value [network.eth0.tx] type [integer]"
3437+
)
3438+
);
3439+
3440+
assertThat(
3441+
error("TS k8s | STATS rate(network.eth0.tx) by tbucket(1m)", k8s),
3442+
equalTo(
3443+
"1:16: first argument of [rate(network.eth0.tx)] must be [counter_long, counter_integer or counter_double], "
3444+
+ "found value [network.eth0.tx] type [integer]"
3445+
)
3446+
);
34263447

3448+
assertThat(
3449+
error("TS k8s | STATS avg(rate(network.eth0.tx))", k8s),
3450+
equalTo(
3451+
"1:20: first argument of [rate(network.eth0.tx)] must be [counter_long, counter_integer or counter_double], "
3452+
+ "found value [network.eth0.tx] type [integer]"
3453+
)
3454+
);
34273455
}
34283456

34293457
public void testMvIntersectionValidatesDataTypesAreEqual() {

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/TimeSeriesBareAggregationsTests.java‎

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -99,13 +99,10 @@ public void testBareAvgOverTime() {
9999
assertThat("Should have aggregates", aggregates.isEmpty(), is(false));
100100

101101
List<Attribute> output = plan.output();
102-
boolean hasTimeseries = output.stream().anyMatch(attr -> attr.name().equals(MetadataAttribute.TIMESERIES));
102+
boolean hasTimeseries = output.stream().anyMatch(MetadataAttribute::isTimeSeriesAttribute);
103103
assertThat("Should have _timeseries in output", hasTimeseries, is(true));
104104

105-
Attribute timeseriesAttr = output.stream()
106-
.filter(attr -> attr.name().equals(MetadataAttribute.TIMESERIES))
107-
.findFirst()
108-
.orElse(null);
105+
Attribute timeseriesAttr = output.stream().filter(MetadataAttribute::isTimeSeriesAttribute).findFirst().orElse(null);
109106

110107
assertNotNull(timeseriesAttr);
111108
assertThat("_timeseries attribute should exist", timeseriesAttr, is(instanceOf(Attribute.class)));
@@ -133,13 +130,10 @@ public void testBareSumOverTime() {
133130
assertThat("Should have aggregates", aggregates.isEmpty(), is(false));
134131

135132
List<Attribute> output = plan.output();
136-
boolean hasTimeseries = output.stream().anyMatch(attr -> attr.name().equals(MetadataAttribute.TIMESERIES));
133+
boolean hasTimeseries = output.stream().anyMatch(MetadataAttribute::isTimeSeriesAttribute);
137134
assertThat("Should have _timeseries in output", hasTimeseries, is(true));
138135

139-
Attribute timeseriesAttr = output.stream()
140-
.filter(attr -> attr.name().equals(MetadataAttribute.TIMESERIES))
141-
.findFirst()
142-
.orElse(null);
136+
Attribute timeseriesAttr = output.stream().filter(MetadataAttribute::isTimeSeriesAttribute).findFirst().orElse(null);
143137

144138
assertNotNull(timeseriesAttr);
145139
assertThat("_timeseries attribute should exist", timeseriesAttr, is(instanceOf(Attribute.class)));
@@ -170,13 +164,10 @@ public void testSumOverTimeWithTBucket() {
170164
assertThat("Should group by bucket", hasBucket, is(true));
171165

172166
List<Attribute> output = plan.output();
173-
boolean hasTimeseries = output.stream().anyMatch(attr -> attr.name().equals(MetadataAttribute.TIMESERIES));
167+
boolean hasTimeseries = output.stream().anyMatch(MetadataAttribute::isTimeSeriesAttribute);
174168
assertThat("Should have _timeseries in output", hasTimeseries, is(true));
175169

176-
Attribute timeseriesAttr = output.stream()
177-
.filter(attr -> attr.name().equals(MetadataAttribute.TIMESERIES))
178-
.findFirst()
179-
.orElse(null);
170+
Attribute timeseriesAttr = output.stream().filter(MetadataAttribute::isTimeSeriesAttribute).findFirst().orElse(null);
180171

181172
assertNotNull(timeseriesAttr);
182173
assertThat("_timeseries attribute should exist", timeseriesAttr, is(instanceOf(Attribute.class)));
@@ -207,13 +198,10 @@ public void testRateWithTBucket() {
207198
assertThat("Should group by bucket", hasBucket, is(true));
208199

209200
List<Attribute> output = plan.output();
210-
boolean hasTimeseries = output.stream().anyMatch(attr -> attr.name().equals(MetadataAttribute.TIMESERIES));
201+
boolean hasTimeseries = output.stream().anyMatch(MetadataAttribute::isTimeSeriesAttribute);
211202
assertThat("Should have _timeseries in output", hasTimeseries, is(true));
212203

213-
Attribute timeseriesAttr = output.stream()
214-
.filter(attr -> attr.name().equals(MetadataAttribute.TIMESERIES))
215-
.findFirst()
216-
.orElse(null);
204+
Attribute timeseriesAttr = output.stream().filter(MetadataAttribute::isTimeSeriesAttribute).findFirst().orElse(null);
217205

218206
assertNotNull(timeseriesAttr);
219207
assertThat("_timeseries attribute should exist", timeseriesAttr, is(instanceOf(Attribute.class)));
@@ -247,13 +235,10 @@ public void testCountOverTime() {
247235
assertThat("Should still group by _tsid", hasTsid, is(true));
248236

249237
List<Attribute> output = plan.output();
250-
boolean hasTimeseries = output.stream().anyMatch(attr -> attr.name().equals(MetadataAttribute.TIMESERIES));
238+
boolean hasTimeseries = output.stream().anyMatch(MetadataAttribute::isTimeSeriesAttribute);
251239
assertThat("Should have _timeseries in output", hasTimeseries, is(true));
252240

253-
Attribute timeseriesAttr = output.stream()
254-
.filter(attr -> attr.name().equals(MetadataAttribute.TIMESERIES))
255-
.findFirst()
256-
.orElse(null);
241+
Attribute timeseriesAttr = output.stream().filter(MetadataAttribute::isTimeSeriesAttribute).findFirst().orElse(null);
257242

258243
assertNotNull(timeseriesAttr);
259244
assertThat("_timeseries attribute should exist", timeseriesAttr, is(instanceOf(Attribute.class)));

0 commit comments

Comments
 (0)