Skip to content

Commit d6c0090

Browse files
JonasKunzJonasKunz
authored
ESQL: Add support for exponential_histogram in median (#138402)
1 parent e57ea35 commit d6c0090

File tree

9 files changed

+101
-70
lines changed

9 files changed

+101
-70
lines changed

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -430,11 +430,11 @@ protected boolean supportsExponentialHistograms() {
430430
try {
431431
return RestEsqlTestCase.hasCapabilities(
432432
client(),
433-
List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V1.capabilityName())
433+
List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName())
434434
)
435435
&& RestEsqlTestCase.hasCapabilities(
436436
remoteClusterClient(),
437-
List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V1.capabilityName())
437+
List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName())
438438
);
439439
} catch (IOException e) {
440440
throw new RuntimeException(e);

‎x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ protected boolean supportsSourceFieldMapping() {
5858
protected boolean supportsExponentialHistograms() {
5959
return RestEsqlTestCase.hasCapabilities(
6060
client(),
61-
List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V1.capabilityName())
61+
List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName())
6262
);
6363
}
6464

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ protected boolean supportsSourceFieldMapping() throws IOException {
289289
protected boolean supportsExponentialHistograms() {
290290
return RestEsqlTestCase.hasCapabilities(
291291
client(),
292-
List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V1.capabilityName())
292+
List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName())
293293
);
294294
}
295295

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec‎

Lines changed: 65 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
loadFiltered
2-
required_capability: exponential_histogram_pre_tech_preview_v1
2+
required_capability: exponential_histogram_pre_tech_preview_v2
33

44
FROM exp_histo_sample | WHERE STARTS_WITH(instance, "dummy") | SORT instance | KEEP instance, responseTime
55
;
@@ -17,71 +17,89 @@ dummy-zero_threshold_only | "{""scale"":0,""zero"":{""threshold"":2.0E-5}}"
1717

1818

1919
allAggsGrouped
20-
required_capability: exponential_histogram_pre_tech_preview_v1
20+
required_capability: exponential_histogram_pre_tech_preview_v2
2121

2222
FROM exp_histo_sample
2323
| EVAL instance = CASE(STARTS_WITH(instance, "dummy"), "dummy-grouped", instance)
24-
| STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75), sum = SUM(responseTime), avg = AVG(responseTime) BY instance
25-
| EVAL p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues
26-
| KEEP instance, min, max, p75, sum, avg
24+
| STATS min = MIN(responseTime), max = MAX(responseTime), median = MEDIAN(responseTime), p75 = PERCENTILE(responseTime,75), sum = SUM(responseTime), avg = AVG(responseTime) BY instance
25+
| EVAL median = ROUND(median, 7), p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues
26+
| KEEP instance, min, max, median, p75, sum, avg
2727
| SORT instance
2828
;
2929

30-
instance:keyword | min:double | max:double | p75:double | sum:double | avg:double
31-
dummy-grouped | -100.0 | 50.0 | 8.3457089 | -7550.0 | -15.0398406374502
32-
instance-0 | 2.4E-4 | 6.786232 | 0.2608237 | 1472.744209 | 0.1665811796176903
33-
instance-1 | 2.17E-4 | 3.190723 | 0.0016068 | 36.198484 | 0.011137995076923077
34-
instance-2 | 2.2E-4 | 2.744054 | 0.0016068 | 27.706021000000003 | 0.008197047633136096
30+
instance:keyword | min:double | max:double | median:double | p75:double | sum:double | avg:double
31+
dummy-grouped | -100.0 | 50.0 | 0.0 | 8.3457089 | -7550.0 | -15.0398406374502
32+
instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 | 1472.744209 | 0.1665811796176903
33+
instance-1 | 2.17E-4 | 3.190723 | 6.469E-4 | 0.0016068 | 36.198484 | 0.011137995076923077
34+
instance-2 | 2.2E-4 | 2.744054 | 6.469E-4 | 0.0016068 | 27.706021000000003 | 0.008197047633136096
3535
;
3636

3737

38+
allAggsFiltered
39+
required_capability: exponential_histogram_pre_tech_preview_v2
40+
41+
FROM exp_histo_sample
42+
| STATS min = MIN(responseTime) WHERE instance == "instance-0",
43+
max = MAX(responseTime) WHERE instance == "instance-1",
44+
median = MEDIAN(responseTime) WHERE instance == "instance-2",
45+
p75 = PERCENTILE(responseTime,75) WHERE instance == "instance-0",
46+
sum = SUM(responseTime) WHERE instance == "instance-1",
47+
avg = AVG(responseTime) WHERE instance == "instance-2"
48+
| EVAL median = ROUND(median, 7), p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues
49+
| KEEP min, max, median, p75, sum, avg
50+
;
51+
52+
min:double | max:double | median:double | p75:double | sum:double | avg:double
53+
2.4E-4 | 3.190723 | 6.469E-4 | 0.2608237 | 36.198484 | 0.008197047633136096
54+
;
55+
3856

3957
allAggsInlineGrouped
40-
required_capability: exponential_histogram_pre_tech_preview_v1
58+
required_capability: exponential_histogram_pre_tech_preview_v2
4159

4260
FROM exp_histo_sample
43-
| INLINE STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75), sum = SUM(responseTime), avg = AVG(responseTime) BY instance
44-
| EVAL p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues
45-
| KEEP instance, min, max, p75, sum , avg
61+
| INLINE STATS min = MIN(responseTime), max = MAX(responseTime), median = MEDIAN(responseTime), p75 = PERCENTILE(responseTime,75), sum = SUM(responseTime), avg = AVG(responseTime) BY instance
62+
| EVAL median = ROUND(median, 7), p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues
63+
| KEEP instance, min, max, median, p75, sum , avg
4664
| SORT instance
4765
| Limit 15
4866
;
4967

50-
instance:keyword | min:double | max:double | p75:double | sum:double | avg:double
51-
dummy-empty | null | null | null | null | null
52-
dummy-full | -100.0 | 50.0 | 10.6666667 | -3775.0 | -25.0
53-
dummy-negative_only | -50.0 | -1.0 | -12.8729318 | -1275.0 | -25.5
54-
dummy-no_zero_bucket | -100.0 | 50.0 | 10.6666667 | -3775.0 | -25.166666666666668
55-
dummy-positive_only | 1.0 | 50.0 | 34.7656715 | 1275.0 | 25.5
56-
dummy-zero_count_only | 0.0 | 0.0 | 0.0 | 0.0 | 0.0
57-
dummy-zero_threshold_only | null | null | null | null | null
58-
instance-0 | 2.4E-4 | 6.786232 | 0.2608237 | 1472.744209 | 0.1665811796176903
59-
instance-0 | 2.4E-4 | 6.786232 | 0.2608237 | 1472.744209 | 0.1665811796176903
60-
instance-0 | 2.4E-4 | 6.786232 | 0.2608237 | 1472.744209 | 0.1665811796176903
61-
instance-0 | 2.4E-4 | 6.786232 | 0.2608237 | 1472.744209 | 0.1665811796176903
62-
instance-0 | 2.4E-4 | 6.786232 | 0.2608237 | 1472.744209 | 0.1665811796176903
63-
instance-0 | 2.4E-4 | 6.786232 | 0.2608237 | 1472.744209 | 0.1665811796176903
64-
instance-0 | 2.4E-4 | 6.786232 | 0.2608237 | 1472.744209 | 0.1665811796176903
65-
instance-0 | 2.4E-4 | 6.786232 | 0.2608237 | 1472.744209 | 0.1665811796176903
68+
instance:keyword | min:double | max:double | median:double | p75:double | sum:double | avg:double
69+
dummy-empty | null | null | null | null | null | null
70+
dummy-full | -100.0 | 50.0 | -21.3333333 | 10.6666667 | -3775.0 | -25.0
71+
dummy-negative_only | -50.0 | -1.0 | -24.5830421 | -12.8729318 | -1275.0 | -25.5
72+
dummy-no_zero_bucket | -100.0 | 50.0 | -21.3333333 | 10.6666667 | -3775.0 | -25.166666666666668
73+
dummy-positive_only | 1.0 | 50.0 | 24.5830421 | 34.7656715 | 1275.0 | 25.5
74+
dummy-zero_count_only | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0
75+
dummy-zero_threshold_only | null | null | null | null | null | null
76+
instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 | 1472.744209 | 0.1665811796176903
77+
instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 | 1472.744209 | 0.1665811796176903
78+
instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 | 1472.744209 | 0.1665811796176903
79+
instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 | 1472.744209 | 0.1665811796176903
80+
instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 | 1472.744209 | 0.1665811796176903
81+
instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 | 1472.744209 | 0.1665811796176903
82+
instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 | 1472.744209 | 0.1665811796176903
83+
instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 | 1472.744209 | 0.1665811796176903
6684
;
6785

6886

6987

7088
allAggsOnEmptyHistogram
71-
required_capability: exponential_histogram_pre_tech_preview_v1
89+
required_capability: exponential_histogram_pre_tech_preview_v2
7290

7391
FROM exp_histo_sample | WHERE instance == "dummy-empty"
74-
| STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75)
75-
| KEEP min, max, p75
92+
| STATS min = MIN(responseTime), max = MAX(responseTime), median = MEDIAN(responseTime), p75 = PERCENTILE(responseTime,75), sum = SUM(responseTime), avg = AVG(responseTime)
93+
| KEEP min, max, median, p75, sum, avg
7694
;
7795

78-
min:double | max:double | p75:double
79-
NULL | NULL | NULL
96+
min:double | max:double | median:double | p75:double | sum:double | avg:double
97+
NULL | NULL | NULL | NULL | NULL | NULL
8098
;
8199

82100

83101
histoAsCaseValue
84-
required_capability: exponential_histogram_pre_tech_preview_v1
102+
required_capability: exponential_histogram_pre_tech_preview_v2
85103

86104
FROM exp_histo_sample
87105
| INLINE STATS p50 = PERCENTILE(responseTime, 50) BY instance, @timestamp
@@ -95,7 +113,7 @@ filteredCount:long
95113
;
96114

97115
ungroupedPercentiles
98-
required_capability: exponential_histogram_pre_tech_preview_v1
116+
required_capability: exponential_histogram_pre_tech_preview_v2
99117

100118
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
101119
| STATS p0 = PERCENTILE(responseTime,0), p50 = PERCENTILE(responseTime,50), p99 = PERCENTILE(responseTime, 99), p100 = PERCENTILE(responseTime,100)
@@ -110,7 +128,7 @@ p0:double | p50:double | p99:double | p100:double
110128

111129

112130
groupedPercentiles
113-
required_capability: exponential_histogram_pre_tech_preview_v1
131+
required_capability: exponential_histogram_pre_tech_preview_v2
114132

115133
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
116134
| STATS p0 = PERCENTILE(responseTime,0), p50 = PERCENTILE(responseTime,50), p99 = PERCENTILE(responseTime, 99), p100 = PERCENTILE(responseTime,100) BY instance
@@ -128,7 +146,7 @@ instance-2 | 2.2E-4 | 6.469E-4 | 0.0857672 | 2.7059714542564097
128146

129147

130148
percentileOnEmptyHistogram
131-
required_capability: exponential_histogram_pre_tech_preview_v1
149+
required_capability: exponential_histogram_pre_tech_preview_v2
132150

133151
FROM exp_histo_sample | WHERE instance == "dummy-empty"
134152
| STATS p50 = PERCENTILE(responseTime,50)
@@ -142,7 +160,7 @@ NULL
142160

143161

144162
ungroupedMinMax
145-
required_capability: exponential_histogram_pre_tech_preview_v1
163+
required_capability: exponential_histogram_pre_tech_preview_v2
146164

147165
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
148166
| STATS min = MIN(responseTime), max = MAX(responseTime)
@@ -156,7 +174,7 @@ min:double | max:double
156174

157175

158176
groupedMinMax
159-
required_capability: exponential_histogram_pre_tech_preview_v1
177+
required_capability: exponential_histogram_pre_tech_preview_v2
160178

161179
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
162180
| STATS min = MIN(responseTime), max = MAX(responseTime) BY instance
@@ -173,7 +191,7 @@ instance-2 | 2.2E-4 | 2.744054
173191

174192

175193
minMaxOnEmptyHistogram
176-
required_capability: exponential_histogram_pre_tech_preview_v1
194+
required_capability: exponential_histogram_pre_tech_preview_v2
177195

178196
FROM exp_histo_sample | WHERE instance == "dummy-empty"
179197
| STATS min = MIN(responseTime), max = MAX(responseTime)
@@ -186,7 +204,7 @@ NULL | NULL
186204

187205

188206
ungroupedAvg
189-
required_capability: exponential_histogram_pre_tech_preview_v1
207+
required_capability: exponential_histogram_pre_tech_preview_v2
190208

191209
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
192210
| STATS avg = AVG(responseTime)
@@ -199,7 +217,7 @@ avg:double
199217

200218

201219
groupedAvg
202-
required_capability: exponential_histogram_pre_tech_preview_v1
220+
required_capability: exponential_histogram_pre_tech_preview_v2
203221

204222
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
205223
| STATS avg = AVG(responseTime) BY instance
@@ -215,7 +233,7 @@ instance-2 | 0.008197047633136096
215233

216234

217235
avgOnEmptyHistogram
218-
required_capability: exponential_histogram_pre_tech_preview_v1
236+
required_capability: exponential_histogram_pre_tech_preview_v2
219237

220238
FROM exp_histo_sample | WHERE instance == "dummy-empty"
221239
| STATS avg = AVG(responseTime)
@@ -228,7 +246,7 @@ NULL
228246

229247

230248
ungroupedSum
231-
required_capability: exponential_histogram_pre_tech_preview_v1
249+
required_capability: exponential_histogram_pre_tech_preview_v2
232250

233251
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
234252
| STATS sum = SUM(responseTime)
@@ -241,7 +259,7 @@ sum:double
241259

242260

243261
groupedSum
244-
required_capability: exponential_histogram_pre_tech_preview_v1
262+
required_capability: exponential_histogram_pre_tech_preview_v2
245263

246264
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
247265
| STATS sum = SUM(responseTime) BY instance

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1558,7 +1558,7 @@ public enum Cap {
15581558
* When implementing changes on this type, we'll simply increment the version suffix at the end to prevent bwc tests from running.
15591559
* As soon as we move into tech preview, we'll replace this capability with a "EXPONENTIAL_HISTOGRAM_TECH_PREVIEW" one.
15601560
*/
1561-
EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V1(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG),
1561+
EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG),
15621562

15631563
/**
15641564
* Create new block when filtering OrdinalBytesRefBlock

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public Median(
5959
Source source,
6060
@Param(
6161
name = "number",
62-
type = { "double", "integer", "long" },
62+
type = { "double", "integer", "long", "exponential_histogram" },
6363
description = "Expression that outputs values to calculate the median of."
6464
) Expression field
6565
) {
@@ -74,9 +74,10 @@ public Median(Source source, Expression field, Expression filter, Expression win
7474
protected Expression.TypeResolution resolveType() {
7575
return isType(
7676
field(),
77-
dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG,
77+
dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG || dt == DataType.EXPONENTIAL_HISTOGRAM,
7878
sourceText(),
7979
DEFAULT,
80+
"exponential_histogram",
8081
"numeric except unsigned_long or counter types"
8182
);
8283
}
@@ -115,7 +116,7 @@ public Expression surrogate() {
115116
var s = source();
116117
var field = field();
117118

118-
return field.foldable()
119+
return field.foldable() && field.dataType() != DataType.EXPONENTIAL_HISTOGRAM
119120
? new MvMedian(s, new ToDouble(s, field))
120121
: new Percentile(source(), field(), filter(), window(), new Literal(source(), (int) QuantileStates.MEDIAN, DataType.INTEGER));
121122
}

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2050,7 +2050,7 @@ public void testUnsupportedTypesInStats() {
20502050
found value [x] type [unsigned_long]
20512051
line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long, _source, or counter types],\
20522052
found value [x] type [unsigned_long]
2053-
line 2:47: argument of [median(x)] must be [numeric except unsigned_long or counter types],\
2053+
line 2:47: argument of [median(x)] must be [exponential_histogram or numeric except unsigned_long or counter types],\
20542054
found value [x] type [unsigned_long]
20552055
line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\
20562056
found value [x] type [unsigned_long]
@@ -2068,7 +2068,7 @@ public void testUnsupportedTypesInStats() {
20682068
line 2:10: argument of [avg(x)] must be [aggregate_metric_double,\
20692069
exponential_histogram or numeric except unsigned_long or counter types],\
20702070
found value [x] type [version]
2071-
line 2:18: argument of [median(x)] must be [numeric except unsigned_long or counter types],\
2071+
line 2:18: argument of [median(x)] must be [exponential_histogram or numeric except unsigned_long or counter types],\
20722072
found value [x] type [version]
20732073
line 2:29: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\
20742074
found value [x] type [version]

0 commit comments

Comments
 (0)