Skip to content

Add support to VALUES aggregation for spatial types #122886

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Feb 25, 2025
Merged

Add support to VALUES aggregation for spatial types #122886

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
cb38008
Add support to VALUES aggregation for spatial types
craigtaverner Feb 18, 2025
bedd45b
Update docs/changelog/122886.yaml
craigtaverner Feb 18, 2025
2f0b166
Merge branch 'main' into agg_values_spatial
craigtaverner Feb 18, 2025
7ab9e7a
Add spatial types to values tests and regenerated docs
craigtaverner Feb 19, 2025
186bf2a
Merge branch 'agg_values_spatial' of github.com:craigtaverner/elastic…
craigtaverner Feb 19, 2025
6171431
Merge branch 'main' into agg_values_spatial
craigtaverner Feb 19, 2025
6ef6510
Rest compatibility should not assert unsupported values
craigtaverner Feb 19, 2025
075f2c0
Merge remote-tracking branch 'origin/main' into agg_values_spatial
craigtaverner Feb 19, 2025
061ca59
Remove asciidoc changes and disable further asciidoc generation
craigtaverner Feb 19, 2025
c7376fc
Add comment about disabling docs generation
craigtaverner Feb 21, 2025
3da53b7
Merge branch 'main' into agg_values_spatial
craigtaverner Feb 21, 2025
f9264c9
Merge branch 'main' into agg_values_spatial
craigtaverner Feb 21, 2025
e66dfeb
Merge branch 'main' into agg_values_spatial
craigtaverner Feb 21, 2025
a4849ea
Make tests less sensitive to ordering issues in serverless
craigtaverner Feb 21, 2025
236347e
Merge branch 'agg_values_spatial' of github.com:craigtaverner/elastic…
craigtaverner Feb 21, 2025
433e0df
Merge branch 'main' into agg_values_spatial
craigtaverner Feb 21, 2025
c1fea0c
Merge branch 'main' into agg_values_spatial
craigtaverner Feb 21, 2025
6513317
Merge branch 'main' into agg_values_spatial
craigtaverner Feb 24, 2025
2ded426
Make tests less sensitive to ordering issues in serverless
craigtaverner Feb 25, 2025
59733e2
Merge branch 'agg_values_spatial' of github.com:craigtaverner/elastic…
craigtaverner Feb 25, 2025
7ef9322
Merge remote-tracking branch 'origin/main' into agg_values_spatial
craigtaverner Feb 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/changelog/122886.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 122886
summary: Add support to VALUES aggregation for spatial types
area: ES|QL
type: bug
issues:
- 122413
61 changes: 61 additions & 0 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,67 @@ c:long | x:double | y:double
19 | null | null
;

values
required_capability: agg_values_spatial

FROM airports
| WHERE scalerank == 9
| STATS locations=VALUES(location)
;

locations:geo_point
["POINT(75.9570722403652 30.8503598561702)", "POINT(75.9330597710755 17.625415183635)", "POINT(85.3235970368767 23.3177245989962)", "POINT(48.7471065435931 31.3431585560757)", "POINT(78.2172186546348 26.285487697937)", "POINT(42.97109630194 14.7552534413725)", "POINT(75.8092915005895 22.727749187571)", "POINT(73.8105674924689 19.9660205672806)", "POINT(76.8017261105242 30.6707248949667)", "POINT(75.3958432922005 19.8672969621082)", "POINT(72.9878190922305 31.3627435480862)", "POINT(73.3163595376585 54.9576482934059)", "POINT(82.6671524525865 55.0095847136264)", "POINT(35.3018728575279 47.8732635579023)", "POINT(101.446569298441 0.464600872998505)", "POINT(145.243980298582 14.1717712971216)", "POINT(73.4084964764375 61.3401672194481)", "POINT(78.7089578747476 10.7603571306554)", "POINT(63.0279333519181 25.988794590011)", "POINT(66.9487311480949 30.249043186181)", "POINT(60.900708564915 29.4752941956573)", "POINT(112.711418617258 -7.92998002840567)", "POINT(83.5504532124038 53.3633850813046)", "POINT(75.7584828456005 31.4329422397715)", "POINT(79.452002687657 28.4218087161144)", "POINT(73.0320498392002 33.5614146278861)", "POINT(127.495916124681 36.7220227766673)", "POINT(126.810839481226 35.1400051390198)", "POINT(128.637537699933 35.8999277969087)", "POINT(129.355731047528 35.5928957527107)", "POINT(105.176060419161 -5.242566777132)", "POINT(81.7317271462187 25.443522027821)", "POINT(61.5122589740201 55.2977919496055)"]
;

valuesGrouped
required_capability: agg_values_spatial

FROM airports
| WHERE scalerank == 9
| EVAL first_letter = SUBSTRING(abbrev, 0, 1)
| STATS locations=VALUES(location) BY first_letter
| SORT first_letter
| KEEP first_letter, locations
;

first_letter:keyword | locations:geo_point
A | POINT (48.7471065435931 31.3431585560757)
B | POINT (83.5504532124038 53.3633850813046)
C | [POINT (127.495916124681 36.7220227766673), POINT (61.5122589740201 55.2977919496055)]
G | POINT (78.2172186546348 26.285487697937)
H | POINT (42.97109630194 14.7552534413725)
I | [POINT (85.3235970368767 23.3177245989962), POINT (75.8092915005895 22.727749187571), POINT (73.8105674924689 19.9660205672806), POINT (76.8017261105242 30.6707248949667), POINT (75.3958432922005 19.8672969621082), POINT (81.7317271462187 25.443522027821)]
K | POINT (126.810839481226 35.1400051390198)
L | [POINT (75.9570722403652 30.8503598561702), POINT (72.9878190922305 31.3627435480862)]
M | POINT (112.711418617258 -7.92998002840567)
O | [POINT (73.3163595376585 54.9576482934059), POINT (82.6671524525865 55.0095847136264), POINT (35.3018728575279 47.8732635579023), POINT (73.0320498392002 33.5614146278861)]
P | POINT (101.446569298441 0.464600872998505)
R | POINT (145.243980298582 14.1717712971216)
S | [POINT (75.9330597710755 17.625415183635), POINT (73.4084964764375 61.3401672194481)]
T | [POINT (78.7089578747476 10.7603571306554), POINT (63.0279333519181 25.988794590011), POINT (128.637537699933 35.8999277969087)]
U | [POINT (66.9487311480949 30.249043186181), POINT (129.355731047528 35.5928957527107)]
V | [POINT (75.7584828456005 31.4329422397715), POINT (79.452002687657 28.4218087161144)]
W | POINT (105.176060419161 -5.242566777132)
Z | POINT (60.900708564915 29.4752941956573)
;

valuesGroupedByOrdinals
required_capability: agg_values_spatial

FROM airports
| WHERE scalerank == 9
| STATS locations=VALUES(location) BY type
| SORT type
| KEEP type, locations
;

type:keyword | locations:geo_point
major | [POINT (76.8017261105242 30.6707248949667), POINT (127.495916124681 36.7220227766673)]
mid | [POINT (75.9330597710755 17.625415183635), POINT (85.3235970368767 23.3177245989962), POINT (48.7471065435931 31.3431585560757), POINT (78.2172186546348 26.285487697937), POINT (42.97109630194 14.7552534413725), POINT (75.8092915005895 22.727749187571), POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (72.9878190922305 31.3627435480862), POINT (73.3163595376585 54.9576482934059), POINT (82.6671524525865 55.0095847136264), POINT (35.3018728575279 47.8732635579023), POINT (101.446569298441 0.464600872998505), POINT (145.243980298582 14.1717712971216), POINT (73.4084964764375 61.3401672194481), POINT (78.7089578747476 10.7603571306554), POINT (63.0279333519181 25.988794590011), POINT (66.9487311480949 30.249043186181), POINT (60.900708564915 29.4752941956573), POINT (112.711418617258 -7.92998002840567), POINT (83.5504532124038 53.3633850813046), POINT (75.7584828456005 31.4329422397715), POINT (126.810839481226 35.1400051390198), POINT (128.637537699933 35.8999277969087), POINT (129.355731047528 35.5928957527107), POINT (105.176060419161 -5.242566777132), POINT (61.5122589740201 55.2977919496055)]
military | [POINT (78.2172186546348 26.285487697937), POINT (76.8017261105242 30.6707248949667), POINT (72.9878190922305 31.3627435480862), POINT (35.3018728575279 47.8732635579023), POINT (112.711418617258 -7.92998002840567), POINT (75.7584828456005 31.4329422397715), POINT (79.452002687657 28.4218087161144), POINT (126.810839481226 35.1400051390198), POINT (81.7317271462187 25.443522027821)]
small | [POINT (75.9570722403652 30.8503598561702), POINT (73.0320498392002 33.5614146278861)]
;

###############################################
# Tests for ST_CENTROID_AGG on GEO_POINT type

Expand Down
5 changes: 5 additions & 0 deletions x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ public enum Cap {
*/
AGG_VALUES,

/**
* Expand the {@code VALUES} agg to cover spatial types.
*/
AGG_VALUES_SPATIAL,

/**
* Does ESQL support async queries.
*/
Expand Down
11 changes: 0 additions & 11 deletions ...lugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,6 @@ public static Expression.TypeResolution isExact(Expression e, String operationNa
GEO_SHAPE.typeName(),
CARTESIAN_SHAPE.typeName() };
private static final String[] POINT_TYPE_NAMES = new String[] { GEO_POINT.typeName(), CARTESIAN_POINT.typeName() };
private static final String[] NON_SPATIAL_TYPE_NAMES = DataType.types()
.stream()
.filter(DataType::isRepresentable)
.filter(t -> DataType.isSpatial(t) == false)
.map(DataType::esType)
.toArray(String[]::new);

public static Expression.TypeResolution isSpatialPoint(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
return isType(e, DataType::isSpatialPoint, operationName, paramOrd, POINT_TYPE_NAMES);
Expand All @@ -77,9 +71,4 @@ public static Expression.TypeResolution isSpatialPoint(Expression e, String oper
public static Expression.TypeResolution isSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
return isType(e, DataType::isSpatial, operationName, paramOrd, SPATIAL_TYPE_NAMES);
}

public static Expression.TypeResolution isNotSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
Copy link
Contributor Author

@craigtaverner craigtaverner Feb 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function was never used anyway

return isType(e, t -> DataType.isSpatial(t) == false, operationName, paramOrd, NON_SPATIAL_TYPE_NAMES);
}

}
43 changes: 34 additions & 9 deletions ...esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,28 @@ public class Values extends AggregateFunction implements ToAggregator {
Map.entry(DataType.SEMANTIC_TEXT, ValuesBytesRefAggregatorFunctionSupplier::new),
Map.entry(DataType.IP, ValuesBytesRefAggregatorFunctionSupplier::new),
Map.entry(DataType.VERSION, ValuesBytesRefAggregatorFunctionSupplier::new),
Map.entry(DataType.GEO_POINT, ValuesBytesRefAggregatorFunctionSupplier::new),
Map.entry(DataType.CARTESIAN_POINT, ValuesBytesRefAggregatorFunctionSupplier::new),
Map.entry(DataType.GEO_SHAPE, ValuesBytesRefAggregatorFunctionSupplier::new),
Map.entry(DataType.CARTESIAN_SHAPE, ValuesBytesRefAggregatorFunctionSupplier::new),
Map.entry(DataType.BOOLEAN, ValuesBooleanAggregatorFunctionSupplier::new)
);

@FunctionInfo(
returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "version" },
returnType = {
"boolean",
"cartesian_point",
"cartesian_shape",
"date",
"date_nanos",
"double",
"geo_point",
"geo_shape",
"integer",
"ip",
"keyword",
"long",
"version" },
preview = true,
description = "Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. "
+ "If you need the values returned in order use <<esql-mv_sort>>.",
Expand All @@ -74,7 +91,21 @@ public Values(
Source source,
@Param(
name = "field",
type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" }
type = {
"boolean",
"cartesian_point",
"cartesian_shape",
"date",
"date_nanos",
"double",
"geo_point",
"geo_shape",
"integer",
"ip",
"keyword",
"long",
"text",
"version" }
) Expression v
) {
this(source, v, Literal.TRUE);
Expand Down Expand Up @@ -115,13 +146,7 @@ public DataType dataType() {

@Override
protected TypeResolution resolveType() {
return TypeResolutions.isType(
field(),
SUPPLIERS::containsKey,
sourceText(),
DEFAULT,
"any type except unsigned_long and spatial types"
);
return TypeResolutions.isType(field(), SUPPLIERS::containsKey, sourceText(), DEFAULT, "any type except unsigned_long");
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion ...est/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,6 @@ protected Expression build(Source source, List<Expression> args) {

@Override
protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except unsigned_long and spatial types"));
return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except unsigned_long"));
}
}
26 changes: 16 additions & 10 deletions x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,19 +149,25 @@ geo_point unsortable with limit from row:

---
values unsupported for geo_point:
- requires:
capabilities:
- method: POST
path: /_query
parameters: [ method, path, parameters, capabilities ]
capabilities: [ agg_values_spatial ]
reason: "Spatial types added to values aggregation in 8.19.0"
test_runner_features: [ capabilities, allowed_warnings_regex ]
- do:
catch: '/.+argument of \[VALUES\(location\)\] must be .+/'
esql.query:
body:
query: 'FROM geo_points | STATS VALUES(location)'

---
values unsupported for geo_point status code:
- do:
catch: bad_request
allowed_warnings_regex:
- "No limit defined, adding default limit of \\[.*\\]"
esql.query:
body:
query: 'FROM geo_points | STATS VALUES(location)'
query: 'FROM geo_points | STATS locations = VALUES(location)'
- length: { columns: 1 }
- match: { columns.0.name: locations }
- match: { columns.0.type: geo_point }
- length: { values: 1 }
- match: { values.0.0: ["POINT (1.0 -1.0)", "POINT (-1.0 1.0)"] }

---
cartesian_point:
Expand Down