Skip to content

Commit fb0f93c

Browse files
craigtavernercraigtaverner
authored
[8.19] Add support to VALUES aggregation for spatial types (#122886) (#127627)
* Add support to VALUES aggregation for spatial types (#122886) The original work at #106065 did not support geospatial types with this comment: > I made this work for everything but geo_point and cartesian_point because I'm not 100% sure how to integrate with those. We can grab those in a follow up. The geospatial types should be possible to collect using the VALUES aggregation with similar behavior to the `ST_COLLECT` OGC function, based on the Elasticsearch convention that treats multi-value geospatial fields as behaving similarly to any geometry collection. So this implementation is a trivial addition to the existing values types support. * Fix docs
1 parent 901ebc2 commit fb0f93c

File tree

11 files changed

+187
-34
lines changed

11 files changed

+187
-34
lines changed

‎docs/changelog/122886.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 122886
2+
summary: Add support to VALUES aggregation for spatial types
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 122413

‎docs/reference/esql/functions/kibana/definition/values.json‎

Lines changed: 48 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/reference/esql/functions/types/values.asciidoc‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎x-pack/plugin/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,8 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
218218
task.skipTest("esql/190_lookup_join/alias-pattern-multiple", "LOOKUP JOIN does not support index aliases for now")
219219
task.skipTest("esql/190_lookup_join/alias-pattern-single", "LOOKUP JOIN does not support index aliases for now")
220220
task.skipTest("esql/180_match_operator/match with disjunctions", "Disjunctions in full text functions work now")
221+
task.skipTest("esql/130_spatial/values unsupported for geo_point", "Spatial types are now supported in VALUES aggregation")
222+
task.skipTest("esql/130_spatial/values unsupported for geo_point status code", "Spatial types are now supported in VALUES aggregation")
221223
task.skipTest("esql/40_tsdb/from doc with aggregate_metric_double", "TODO: support for subset of metric fields")
222224
task.skipTest("esql/40_tsdb/stats on aggregate_metric_double", "TODO: support for subset of metric fields")
223225
task.skipTest("esql/40_tsdb/from index pattern unsupported counter", "TODO: support for subset of metric fields")

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,70 @@ c:long | x:double | y:double
145145
19 | null | null
146146
;
147147

148+
values
149+
required_capability: agg_values_spatial
150+
151+
FROM airports
152+
| WHERE scalerank == 9
153+
| STATS locations=VALUES(location)
154+
| EVAL locations = MV_SORT(TO_STRING(locations))
155+
;
156+
157+
locations:keyword
158+
[POINT (101.446569298441 0.464600872998505), POINT (105.176060419161 -5.242566777132), POINT (112.711418617258 -7.92998002840567), POINT (126.810839481226 35.1400051390198), POINT (127.495916124681 36.7220227766673), POINT (128.637537699933 35.8999277969087), POINT (129.355731047528 35.5928957527107), POINT (145.243980298582 14.1717712971216), POINT (35.3018728575279 47.8732635579023), POINT (42.97109630194 14.7552534413725), POINT (48.7471065435931 31.3431585560757), POINT (60.900708564915 29.4752941956573), POINT (61.5122589740201 55.2977919496055), POINT (63.0279333519181 25.988794590011), POINT (66.9487311480949 30.249043186181), POINT (72.9878190922305 31.3627435480862), POINT (73.0320498392002 33.5614146278861), POINT (73.3163595376585 54.9576482934059), POINT (73.4084964764375 61.3401672194481), POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (75.7584828456005 31.4329422397715), POINT (75.8092915005895 22.727749187571), POINT (75.9330597710755 17.625415183635), POINT (75.9570722403652 30.8503598561702), POINT (76.8017261105242 30.6707248949667), POINT (78.2172186546348 26.285487697937), POINT (78.7089578747476 10.7603571306554), POINT (79.452002687657 28.4218087161144), POINT (81.7317271462187 25.443522027821), POINT (82.6671524525865 55.0095847136264), POINT (83.5504532124038 53.3633850813046), POINT (85.3235970368767 23.3177245989962)]
159+
;
160+
161+
valuesGrouped
162+
required_capability: agg_values_spatial
163+
164+
FROM airports
165+
| WHERE scalerank == 9
166+
| EVAL first_letter = SUBSTRING(abbrev, 0, 1)
167+
| STATS locations=VALUES(location) BY first_letter
168+
| EVAL locations = MV_SORT(TO_STRING(locations))
169+
| SORT first_letter
170+
| KEEP first_letter, locations
171+
;
172+
173+
first_letter:keyword | locations:keyword
174+
A | POINT (48.7471065435931 31.3431585560757)
175+
B | POINT (83.5504532124038 53.3633850813046)
176+
C | [POINT (127.495916124681 36.7220227766673), POINT (61.5122589740201 55.2977919496055)]
177+
G | POINT (78.2172186546348 26.285487697937)
178+
H | POINT (42.97109630194 14.7552534413725)
179+
I | [POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (75.8092915005895 22.727749187571), POINT (76.8017261105242 30.6707248949667), POINT (81.7317271462187 25.443522027821), POINT (85.3235970368767 23.3177245989962)]
180+
K | POINT (126.810839481226 35.1400051390198)
181+
L | [POINT (72.9878190922305 31.3627435480862), POINT (75.9570722403652 30.8503598561702)]
182+
M | POINT (112.711418617258 -7.92998002840567)
183+
O | [POINT (35.3018728575279 47.8732635579023), POINT (73.0320498392002 33.5614146278861), POINT (73.3163595376585 54.9576482934059), POINT (82.6671524525865 55.0095847136264)]
184+
P | POINT (101.446569298441 0.464600872998505)
185+
R | POINT (145.243980298582 14.1717712971216)
186+
S | [POINT (73.4084964764375 61.3401672194481), POINT (75.9330597710755 17.625415183635)]
187+
T | [POINT (128.637537699933 35.8999277969087), POINT (63.0279333519181 25.988794590011), POINT (78.7089578747476 10.7603571306554)]
188+
U | [POINT (129.355731047528 35.5928957527107), POINT (66.9487311480949 30.249043186181)]
189+
V | [POINT (75.7584828456005 31.4329422397715), POINT (79.452002687657 28.4218087161144)]
190+
W | POINT (105.176060419161 -5.242566777132)
191+
Z | POINT (60.900708564915 29.4752941956573)
192+
;
193+
194+
valuesGroupedByOrdinals
195+
required_capability: agg_values_spatial
196+
197+
FROM airports
198+
| WHERE scalerank == 9
199+
| STATS locations=VALUES(location) BY type
200+
| EVAL locations = MV_SORT(TO_STRING(locations))
201+
| SORT type
202+
| KEEP type, locations
203+
;
204+
205+
type:keyword | locations:keyword
206+
major | [POINT (127.495916124681 36.7220227766673), POINT (76.8017261105242 30.6707248949667)]
207+
mid | [POINT (101.446569298441 0.464600872998505), POINT (105.176060419161 -5.242566777132), POINT (112.711418617258 -7.92998002840567), POINT (126.810839481226 35.1400051390198), POINT (128.637537699933 35.8999277969087), POINT (129.355731047528 35.5928957527107), POINT (145.243980298582 14.1717712971216), POINT (35.3018728575279 47.8732635579023), POINT (42.97109630194 14.7552534413725), POINT (48.7471065435931 31.3431585560757), POINT (60.900708564915 29.4752941956573), POINT (61.5122589740201 55.2977919496055), POINT (63.0279333519181 25.988794590011), POINT (66.9487311480949 30.249043186181), POINT (72.9878190922305 31.3627435480862), POINT (73.3163595376585 54.9576482934059), POINT (73.4084964764375 61.3401672194481), POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (75.7584828456005 31.4329422397715), POINT (75.8092915005895 22.727749187571), POINT (75.9330597710755 17.625415183635), POINT (78.2172186546348 26.285487697937), POINT (78.7089578747476 10.7603571306554), POINT (82.6671524525865 55.0095847136264), POINT (83.5504532124038 53.3633850813046), POINT (85.3235970368767 23.3177245989962)]
208+
military | [POINT (112.711418617258 -7.92998002840567), POINT (126.810839481226 35.1400051390198), POINT (35.3018728575279 47.8732635579023), POINT (72.9878190922305 31.3627435480862), POINT (75.7584828456005 31.4329422397715), POINT (76.8017261105242 30.6707248949667), POINT (78.2172186546348 26.285487697937), POINT (79.452002687657 28.4218087161144), POINT (81.7317271462187 25.443522027821)]
209+
small | [POINT (73.0320498392002 33.5614146278861), POINT (75.9570722403652 30.8503598561702)]
210+
;
211+
148212
###############################################
149213
# Tests for ST_CENTROID_AGG on GEO_POINT type
150214

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,11 @@ public enum Cap {
141141
*/
142142
AGG_TOP_STRING_SUPPORT,
143143

144+
/**
145+
* Expand the {@code VALUES} agg to cover spatial types.
146+
*/
147+
AGG_VALUES_SPATIAL,
148+
144149
/**
145150
* {@code CASE} properly handling multivalue conditions.
146151
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,6 @@ public static Expression.TypeResolution isExact(Expression e, String operationNa
6363
GEO_SHAPE.typeName(),
6464
CARTESIAN_SHAPE.typeName() };
6565
private static final String[] POINT_TYPE_NAMES = new String[] { GEO_POINT.typeName(), CARTESIAN_POINT.typeName() };
66-
private static final String[] NON_SPATIAL_TYPE_NAMES = DataType.types()
67-
.stream()
68-
.filter(DataType::isRepresentable)
69-
.filter(t -> DataType.isSpatial(t) == false)
70-
.map(DataType::esType)
71-
.toArray(String[]::new);
7266

7367
public static Expression.TypeResolution isSpatialPoint(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
7468
return isType(e, DataType::isSpatialPoint, operationName, paramOrd, POINT_TYPE_NAMES);
@@ -77,9 +71,4 @@ public static Expression.TypeResolution isSpatialPoint(Expression e, String oper
7771
public static Expression.TypeResolution isSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
7872
return isType(e, DataType::isSpatial, operationName, paramOrd, SPATIAL_TYPE_NAMES);
7973
}
80-
81-
public static Expression.TypeResolution isNotSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
82-
return isType(e, t -> DataType.isSpatial(t) == false, operationName, paramOrd, NON_SPATIAL_TYPE_NAMES);
83-
}
84-
8574
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java‎

Lines changed: 34 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,28 @@ public class Values extends AggregateFunction implements ToAggregator {
4848
Map.entry(DataType.TEXT, ValuesBytesRefAggregatorFunctionSupplier::new),
4949
Map.entry(DataType.IP, ValuesBytesRefAggregatorFunctionSupplier::new),
5050
Map.entry(DataType.VERSION, ValuesBytesRefAggregatorFunctionSupplier::new),
51+
Map.entry(DataType.GEO_POINT, ValuesBytesRefAggregatorFunctionSupplier::new),
52+
Map.entry(DataType.CARTESIAN_POINT, ValuesBytesRefAggregatorFunctionSupplier::new),
53+
Map.entry(DataType.GEO_SHAPE, ValuesBytesRefAggregatorFunctionSupplier::new),
54+
Map.entry(DataType.CARTESIAN_SHAPE, ValuesBytesRefAggregatorFunctionSupplier::new),
5155
Map.entry(DataType.BOOLEAN, ValuesBooleanAggregatorFunctionSupplier::new)
5256
);
5357

5458
@FunctionInfo(
55-
returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "version" },
59+
returnType = {
60+
"boolean",
61+
"cartesian_point",
62+
"cartesian_shape",
63+
"date",
64+
"date_nanos",
65+
"double",
66+
"geo_point",
67+
"geo_shape",
68+
"integer",
69+
"ip",
70+
"keyword",
71+
"long",
72+
"version" },
5673
preview = true,
5774
description = "Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. "
5875
+ "If you need the values returned in order use <<esql-mv_sort>>.",
@@ -72,7 +89,21 @@ public Values(
7289
Source source,
7390
@Param(
7491
name = "field",
75-
type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" }
92+
type = {
93+
"boolean",
94+
"cartesian_point",
95+
"cartesian_shape",
96+
"date",
97+
"date_nanos",
98+
"double",
99+
"geo_point",
100+
"geo_shape",
101+
"integer",
102+
"ip",
103+
"keyword",
104+
"long",
105+
"text",
106+
"version" }
76107
) Expression v
77108
) {
78109
this(source, v, Literal.TRUE);
@@ -113,13 +144,7 @@ public DataType dataType() {
113144

114145
@Override
115146
protected TypeResolution resolveType() {
116-
return TypeResolutions.isType(
117-
field(),
118-
SUPPLIERS::containsKey,
119-
sourceText(),
120-
DEFAULT,
121-
"any type except unsigned_long and spatial types"
122-
);
147+
return TypeResolutions.isType(field(), SUPPLIERS::containsKey, sourceText(), DEFAULT, "any type except unsigned_long");
123148
}
124149

125150
@Override

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,6 @@ protected Expression build(Source source, List<Expression> args) {
3232

3333
@Override
3434
protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
35-
return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except unsigned_long and spatial types"));
35+
return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except unsigned_long"));
3636
}
3737
}

0 commit comments

Comments
 (0)