diff --git a/docs/changelog/122886.yaml b/docs/changelog/122886.yaml new file mode 100644 index 0000000000000..7306a21d1470e --- /dev/null +++ b/docs/changelog/122886.yaml @@ -0,0 +1,6 @@ +pr: 122886 +summary: Add support to VALUES aggregation for spatial types +area: ES|QL +type: bug +issues: + - 122413 diff --git a/docs/reference/esql/functions/kibana/definition/values.json b/docs/reference/esql/functions/kibana/definition/values.json index 95ac402bb242a..0ac74c61cf73d 100644 --- a/docs/reference/esql/functions/kibana/definition/values.json +++ b/docs/reference/esql/functions/kibana/definition/values.json @@ -16,6 +16,30 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "field", + "type" : "cartesian_point", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "cartesian_point" + }, + { + "params" : [ + { + "name" : "field", + "type" : "cartesian_shape", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, { "params" : [ { @@ -52,6 +76,30 @@ "variadic" : false, "returnType" : "double" }, + { + "params" : [ + { + "name" : "field", + "type" : "geo_point", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "geo_point" + }, + { + "params" : [ + { + "name" : "field", + "type" : "geo_shape", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "geo_shape" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/types/values.asciidoc b/docs/reference/esql/functions/types/values.asciidoc index adf457dac31b8..1524ec86cd5ec 100644 --- a/docs/reference/esql/functions/types/values.asciidoc +++ b/docs/reference/esql/functions/types/values.asciidoc @@ -6,9 +6,13 @@ |=== field | result boolean | boolean +cartesian_point | cartesian_point +cartesian_shape | cartesian_shape date | date date_nanos | date_nanos double | double +geo_point | geo_point +geo_shape | geo_shape integer | integer ip | ip keyword | keyword diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index b30de1a5a3870..98164298acd3a 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -218,6 +218,8 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task -> task.skipTest("esql/190_lookup_join/alias-pattern-multiple", "LOOKUP JOIN does not support index aliases for now") task.skipTest("esql/190_lookup_join/alias-pattern-single", "LOOKUP JOIN does not support index aliases for now") task.skipTest("esql/180_match_operator/match with disjunctions", "Disjunctions in full text functions work now") + task.skipTest("esql/130_spatial/values unsupported for geo_point", "Spatial types are now supported in VALUES aggregation") + task.skipTest("esql/130_spatial/values unsupported for geo_point status code", "Spatial types are now supported in VALUES aggregation") task.skipTest("esql/40_tsdb/from doc with aggregate_metric_double", "TODO: support for subset of metric fields") task.skipTest("esql/40_tsdb/stats on aggregate_metric_double", "TODO: support for subset of metric fields") task.skipTest("esql/40_tsdb/from index pattern unsupported counter", "TODO: support for subset of metric fields") diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index d10a6178829e6..e509b82e62781 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -145,6 +145,70 @@ c:long | x:double | y:double 19 | null | null ; +values +required_capability: agg_values_spatial + +FROM airports +| WHERE scalerank == 9 +| STATS locations=VALUES(location) +| EVAL locations = MV_SORT(TO_STRING(locations)) +; + +locations:keyword +[POINT (101.446569298441 0.464600872998505), POINT (105.176060419161 -5.242566777132), POINT (112.711418617258 -7.92998002840567), POINT (126.810839481226 35.1400051390198), POINT (127.495916124681 36.7220227766673), POINT (128.637537699933 35.8999277969087), POINT (129.355731047528 35.5928957527107), POINT (145.243980298582 14.1717712971216), POINT (35.3018728575279 47.8732635579023), POINT (42.97109630194 14.7552534413725), POINT (48.7471065435931 31.3431585560757), POINT (60.900708564915 29.4752941956573), POINT (61.5122589740201 55.2977919496055), POINT (63.0279333519181 25.988794590011), POINT (66.9487311480949 30.249043186181), POINT (72.9878190922305 31.3627435480862), POINT (73.0320498392002 33.5614146278861), POINT (73.3163595376585 54.9576482934059), POINT (73.4084964764375 61.3401672194481), POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (75.7584828456005 31.4329422397715), POINT (75.8092915005895 22.727749187571), POINT (75.9330597710755 17.625415183635), POINT (75.9570722403652 30.8503598561702), POINT (76.8017261105242 30.6707248949667), POINT (78.2172186546348 26.285487697937), POINT (78.7089578747476 10.7603571306554), POINT (79.452002687657 28.4218087161144), POINT (81.7317271462187 25.443522027821), POINT (82.6671524525865 55.0095847136264), POINT (83.5504532124038 53.3633850813046), POINT (85.3235970368767 23.3177245989962)] +; + +valuesGrouped +required_capability: agg_values_spatial + +FROM airports +| WHERE scalerank == 9 +| EVAL first_letter = SUBSTRING(abbrev, 0, 1) +| STATS locations=VALUES(location) BY first_letter +| EVAL locations = MV_SORT(TO_STRING(locations)) +| SORT first_letter +| KEEP first_letter, locations +; + +first_letter:keyword | locations:keyword +A | POINT (48.7471065435931 31.3431585560757) +B | POINT (83.5504532124038 53.3633850813046) +C | [POINT (127.495916124681 36.7220227766673), POINT (61.5122589740201 55.2977919496055)] +G | POINT (78.2172186546348 26.285487697937) +H | POINT (42.97109630194 14.7552534413725) +I | [POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (75.8092915005895 22.727749187571), POINT (76.8017261105242 30.6707248949667), POINT (81.7317271462187 25.443522027821), POINT (85.3235970368767 23.3177245989962)] +K | POINT (126.810839481226 35.1400051390198) +L | [POINT (72.9878190922305 31.3627435480862), POINT (75.9570722403652 30.8503598561702)] +M | POINT (112.711418617258 -7.92998002840567) +O | [POINT (35.3018728575279 47.8732635579023), POINT (73.0320498392002 33.5614146278861), POINT (73.3163595376585 54.9576482934059), POINT (82.6671524525865 55.0095847136264)] +P | POINT (101.446569298441 0.464600872998505) +R | POINT (145.243980298582 14.1717712971216) +S | [POINT (73.4084964764375 61.3401672194481), POINT (75.9330597710755 17.625415183635)] +T | [POINT (128.637537699933 35.8999277969087), POINT (63.0279333519181 25.988794590011), POINT (78.7089578747476 10.7603571306554)] +U | [POINT (129.355731047528 35.5928957527107), POINT (66.9487311480949 30.249043186181)] +V | [POINT (75.7584828456005 31.4329422397715), POINT (79.452002687657 28.4218087161144)] +W | POINT (105.176060419161 -5.242566777132) +Z | POINT (60.900708564915 29.4752941956573) +; + +valuesGroupedByOrdinals +required_capability: agg_values_spatial + +FROM airports +| WHERE scalerank == 9 +| STATS locations=VALUES(location) BY type +| EVAL locations = MV_SORT(TO_STRING(locations)) +| SORT type +| KEEP type, locations +; + +type:keyword | locations:keyword +major | [POINT (127.495916124681 36.7220227766673), POINT (76.8017261105242 30.6707248949667)] +mid | [POINT (101.446569298441 0.464600872998505), POINT (105.176060419161 -5.242566777132), POINT (112.711418617258 -7.92998002840567), POINT (126.810839481226 35.1400051390198), POINT (128.637537699933 35.8999277969087), POINT (129.355731047528 35.5928957527107), POINT (145.243980298582 14.1717712971216), POINT (35.3018728575279 47.8732635579023), POINT (42.97109630194 14.7552534413725), POINT (48.7471065435931 31.3431585560757), POINT (60.900708564915 29.4752941956573), POINT (61.5122589740201 55.2977919496055), POINT (63.0279333519181 25.988794590011), POINT (66.9487311480949 30.249043186181), POINT (72.9878190922305 31.3627435480862), POINT (73.3163595376585 54.9576482934059), POINT (73.4084964764375 61.3401672194481), POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (75.7584828456005 31.4329422397715), POINT (75.8092915005895 22.727749187571), POINT (75.9330597710755 17.625415183635), POINT (78.2172186546348 26.285487697937), POINT (78.7089578747476 10.7603571306554), POINT (82.6671524525865 55.0095847136264), POINT (83.5504532124038 53.3633850813046), POINT (85.3235970368767 23.3177245989962)] +military | [POINT (112.711418617258 -7.92998002840567), POINT (126.810839481226 35.1400051390198), POINT (35.3018728575279 47.8732635579023), POINT (72.9878190922305 31.3627435480862), POINT (75.7584828456005 31.4329422397715), POINT (76.8017261105242 30.6707248949667), POINT (78.2172186546348 26.285487697937), POINT (79.452002687657 28.4218087161144), POINT (81.7317271462187 25.443522027821)] +small | [POINT (73.0320498392002 33.5614146278861), POINT (75.9570722403652 30.8503598561702)] +; + ############################################### # Tests for ST_CENTROID_AGG on GEO_POINT type diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 73cb3e599fef6..1f16bdf2b21eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -141,6 +141,11 @@ public enum Cap { */ AGG_TOP_STRING_SUPPORT, + /** + * Expand the {@code VALUES} agg to cover spatial types. + */ + AGG_VALUES_SPATIAL, + /** * {@code CASE} properly handling multivalue conditions. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java index b97374d179a44..23f875e4a0595 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java @@ -63,12 +63,6 @@ public static Expression.TypeResolution isExact(Expression e, String operationNa GEO_SHAPE.typeName(), CARTESIAN_SHAPE.typeName() }; private static final String[] POINT_TYPE_NAMES = new String[] { GEO_POINT.typeName(), CARTESIAN_POINT.typeName() }; - private static final String[] NON_SPATIAL_TYPE_NAMES = DataType.types() - .stream() - .filter(DataType::isRepresentable) - .filter(t -> DataType.isSpatial(t) == false) - .map(DataType::esType) - .toArray(String[]::new); public static Expression.TypeResolution isSpatialPoint(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { return isType(e, DataType::isSpatialPoint, operationName, paramOrd, POINT_TYPE_NAMES); @@ -77,9 +71,4 @@ public static Expression.TypeResolution isSpatialPoint(Expression e, String oper public static Expression.TypeResolution isSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { return isType(e, DataType::isSpatial, operationName, paramOrd, SPATIAL_TYPE_NAMES); } - - public static Expression.TypeResolution isNotSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { - return isType(e, t -> DataType.isSpatial(t) == false, operationName, paramOrd, NON_SPATIAL_TYPE_NAMES); - } - } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java index 1bc35865530ae..70b0cc9f12b08 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java @@ -48,11 +48,28 @@ public class Values extends AggregateFunction implements ToAggregator { Map.entry(DataType.TEXT, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.IP, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.VERSION, ValuesBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.GEO_POINT, ValuesBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.CARTESIAN_POINT, ValuesBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.GEO_SHAPE, ValuesBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.CARTESIAN_SHAPE, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.BOOLEAN, ValuesBooleanAggregatorFunctionSupplier::new) ); @FunctionInfo( - returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "version" }, + returnType = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "date_nanos", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "version" }, preview = true, description = "Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. " + "If you need the values returned in order use <>.", @@ -72,7 +89,21 @@ public Values( Source source, @Param( name = "field", - type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" } + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "date_nanos", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "version" } ) Expression v ) { this(source, v, Literal.TRUE); @@ -113,13 +144,7 @@ public DataType dataType() { @Override protected TypeResolution resolveType() { - return TypeResolutions.isType( - field(), - SUPPLIERS::containsKey, - sourceText(), - DEFAULT, - "any type except unsigned_long and spatial types" - ); + return TypeResolutions.isType(field(), SUPPLIERS::containsKey, sourceText(), DEFAULT, "any type except unsigned_long"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java index f9dafc954b6f5..d34fe1df29827 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java @@ -32,6 +32,6 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except unsigned_long and spatial types")); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except unsigned_long")); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java index 8aa8411be8b88..da37518a4d76f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java @@ -51,7 +51,12 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.versionCases(1, 1000), // Lower values for strings, as they take more space and may trigger the circuit breaker MultiRowTestCaseSupplier.stringCases(1, 20, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 20, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 20, DataType.TEXT), + // For spatial types, we can have many rows for points, but reduce rows for shapes to avoid circuit breaker + MultiRowTestCaseSupplier.geoPointCases(1, 1000, MultiRowTestCaseSupplier.IncludingAltitude.NO), + MultiRowTestCaseSupplier.cartesianPointCases(1, 1000, MultiRowTestCaseSupplier.IncludingAltitude.NO), + MultiRowTestCaseSupplier.geoShapeCasesWithoutCircle(1, 100, MultiRowTestCaseSupplier.IncludingAltitude.NO), + MultiRowTestCaseSupplier.cartesianShapeCasesWithoutCircle(1, 100, MultiRowTestCaseSupplier.IncludingAltitude.NO) ).flatMap(List::stream).map(ValuesTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, false); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml index ad3adbf93adaf..e632620817283 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml @@ -3,7 +3,7 @@ setup: - requires: cluster_features: ["gte_v8.14.0"] reason: "Mixed cluster tests don't work with the changed error message from sort" - test_runner_features: allowed_warnings_regex + test_runner_features: [ capabilities, allowed_warnings_regex ] - do: indices.create: @@ -148,20 +148,25 @@ geo_point unsortable with limit from row: query: 'ROW wkt = ["POINT(42.9711 -14.7553)", "POINT(75.8093 22.7277)"] | MV_EXPAND wkt | EVAL pt = TO_GEOPOINT(wkt) | limit 5 | sort pt' --- -values unsupported for geo_point: - - do: - catch: '/.+argument of \[VALUES\(location\)\] must be .+/' - esql.query: - body: - query: 'FROM geo_points | STATS VALUES(location)' - ---- -values unsupported for geo_point status code: +values supported for geo_point: + - requires: + capabilities: + - method: POST + path: /_query + parameters: [ method, path, parameters, capabilities ] + capabilities: [ agg_values_spatial ] + reason: "Spatial types added to values aggregation in 8.19.0" - do: - catch: bad_request + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM geo_points | STATS VALUES(location)' + query: 'FROM geo_points | STATS locations = VALUES(location) | EVAL locations = MV_SORT(TO_STRING(locations))' + - length: { columns: 1 } + - match: { columns.0.name: locations } + - match: { columns.0.type: keyword } + - length: { values: 1 } + - match: { values.0.0: ["POINT (-1.0 1.0)", "POINT (1.0 -1.0)"] } --- cartesian_point: