diff --git a/server/src/main/resources/transport/definitions/referable/esql_resolve_fields_response_used.csv b/server/src/main/resources/transport/definitions/referable/esql_resolve_fields_response_used.csv new file mode 100644 index 0000000000000..60121004dc1e2 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_resolve_fields_response_used.csv @@ -0,0 +1 @@ +9204000,9185005 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index e3c62345bb99a..23dfcd8d57f3a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -min_transport_version,9185004 +esql_resolve_fields_response_used,9185005 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index a3e52d8099898..80e35f2bfbc93 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -min_transport_version,9202000 +esql_resolve_fields_response_used,9204000 diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index b05d6784c072f..bff8ab8c7037f 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -976,8 +976,13 @@ Builder underConstruction() { } } - private static class DataTypesTransportVersions { + public static class DataTypesTransportVersions { + /** + * The first transport version after the PR that introduced geotile/geohash/geohex, resp. + * after 9.1. We didn't require transport versions at that point in time, as geotile/hash/hex require + * using specific functions to even occur in query plans. + */ public static final TransportVersion INDEX_SOURCE = TransportVersion.fromName("index_source"); public static final TransportVersion ESQL_DENSE_VECTOR_CREATED_VERSION = TransportVersion.fromName( diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 0617f03402730..c7ae359993bb4 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -44,7 +44,10 @@ import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; -import static org.elasticsearch.xpack.esql.action.EsqlResolveFieldsResponse.RESOLVE_FIELDS_RESPONSE_CREATED_TV; +import static org.elasticsearch.xpack.esql.action.EsqlResolveFieldsResponse.RESOLVE_FIELDS_RESPONSE_USED_TV; +import static org.elasticsearch.xpack.esql.core.type.DataType.DataTypesTransportVersions.ESQL_AGGREGATE_METRIC_DOUBLE_CREATED_VERSION; +import static org.elasticsearch.xpack.esql.core.type.DataType.DataTypesTransportVersions.ESQL_DENSE_VECTOR_CREATED_VERSION; +import static org.elasticsearch.xpack.esql.core.type.DataType.DataTypesTransportVersions.INDEX_SOURCE; import static org.hamcrest.Matchers.any; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; @@ -71,8 +74,6 @@ public class AllSupportedFieldsTestCase extends ESRestTestCase { private static final Logger logger = LogManager.getLogger(FieldExtractorTestCase.class); - private static final TransportVersion INDEX_SOURCE = TransportVersion.fromName("index_source"); - @Rule(order = Integer.MIN_VALUE) public ProfileLogger profileLogger = new ProfileLogger(); @@ -329,6 +330,76 @@ public final void testFetchDenseVector() throws IOException { assertMap(indexToRow(columns, values), expectedAllValues); } + /** + * Tests fetching {@code aggregate_metric_double} if possible. Uses the {@code dense_vector_agg_metric_double_if_fns} + * work around if required. + */ + public final void testFetchAggregateMetricDouble() throws IOException { + Map response; + try { + String request = """ + | EVAL strjunk = TO_STRING(f_aggregate_metric_double) + | KEEP _index, f_aggregate_metric_double + | LIMIT 1000 + """; + if (denseVectorAggMetricDoubleIfVersion() == false) { + request = """ + | EVAL junk = TO_AGGREGATE_METRIC_DOUBLE(1) // workaround to enable fetching aggregate_metric_double + """ + request; + } + response = esql(request); + if ((Boolean) response.get("is_partial")) { + Map clusters = (Map) response.get("_clusters"); + Map details = (Map) clusters.get("details"); + + boolean foundError = false; + for (Map.Entry cluster : details.entrySet()) { + String failures = cluster.getValue().toString(); + if (denseVectorAggMetricDoubleIfFns()) { + throw new AssertionError("should correctly fetch the aggregate_metric_double: " + failures); + } + foundError |= failures.contains("doesn't understand data type [AGGREGATE_METRIC_DOUBLE]"); + } + assertTrue("didn't find errors: " + details, foundError); + return; + } + } catch (ResponseException e) { + if (denseVectorAggMetricDoubleIfFns()) { + throw new AssertionError("should correctly fetch the aggregate_metric_double", e); + } + assertThat( + "old version should fail with this error", + EntityUtils.toString(e.getResponse().getEntity()), + anyOf( + containsString("Unknown function [TO_AGGREGATE_METRIC_DOUBLE]"), + containsString("Cannot use field [f_aggregate_metric_double] with unsupported type"), + containsString("doesn't understand data type [AGGREGATE_METRIC_DOUBLE]") + ) + ); + // Failure is expected and fine + return; + } + List columns = (List) response.get("columns"); + List values = (List) response.get("values"); + + MapMatcher expectedColumns = matchesMap().entry("f_aggregate_metric_double", "aggregate_metric_double").entry("_index", "keyword"); + assertMap(nameToType(columns), expectedColumns); + + MapMatcher expectedAllValues = matchesMap(); + for (Map.Entry e : expectedIndices().entrySet()) { + String indexName = e.getKey(); + NodeInfo nodeInfo = e.getValue(); + MapMatcher expectedValues = matchesMap(); + expectedValues = expectedValues.entry( + "f_aggregate_metric_double", + "{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}" + ); + expectedValues = expectedValues.entry("_index", indexName); + expectedAllValues = expectedAllValues.entry(indexName, expectedValues); + } + assertMap(indexToRow(columns, values), expectedAllValues); + } + private Map esql(String query) throws IOException { Request request = new Request("POST", "_query"); XContentBuilder body = JsonXContent.contentBuilder().startObject(); @@ -473,21 +544,15 @@ private Matcher expectedValue(DataType type, NodeInfo nodeInfo) throws IOExce case GEO_SHAPE -> equalTo("POINT (-71.34 41.12)"); case NULL -> nullValue(); case AGGREGATE_METRIC_DOUBLE -> { - /* - * We need both AGGREGATE_METRIC_DOUBLE_CREATED and RESOLVE_FIELDS_RESPONSE_CREATED_TV - * but RESOLVE_FIELDS_RESPONSE_CREATED_TV came last so it's enough to check just it. - */ - if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_CREATED_TV) == false) { + if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_USED_TV) == false + || minVersion().supports(ESQL_AGGREGATE_METRIC_DOUBLE_CREATED_VERSION) == false) { yield nullValue(); } yield equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); } case DENSE_VECTOR -> { - /* - * We need both DENSE_VECTOR_CREATED and RESOLVE_FIELDS_RESPONSE_CREATED_TV - * but RESOLVE_FIELDS_RESPONSE_CREATED_TV came last so it's enough to check just it. - */ - if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_CREATED_TV) == false) { + if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_USED_TV) == false + || minVersion().supports(ESQL_DENSE_VECTOR_CREATED_VERSION) == false) { yield nullValue(); } yield equalTo(List.of(0.5, 10.0, 5.9999995)); @@ -572,15 +637,24 @@ private Matcher expectedType(DataType type) throws IOException { case BYTE, SHORT -> equalTo("integer"); case HALF_FLOAT, SCALED_FLOAT, FLOAT -> equalTo("double"); case NULL -> equalTo("keyword"); - case AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> { - /* - * We need both _CREATED and RESOLVE_FIELDS_RESPONSE_CREATED_TV - * but RESOLVE_FIELDS_RESPONSE_CREATED_TV came last so it's enough to check just it. - */ - if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_CREATED_TV) == false) { + case AGGREGATE_METRIC_DOUBLE -> { + // RESOLVE_FIELDS_RESPONSE_USED_TV is newer and technically sufficient to check. + // We also check for ESQL_AGGREGATE_METRIC_DOUBLE_CREATED_VERSION for clarity. + // Future data types added here should only require the TV when they were created, + // because it will be after RESOLVE_FIELDS_RESPONSE_USED_TV. + if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_USED_TV) == false + || minVersion().supports(ESQL_AGGREGATE_METRIC_DOUBLE_CREATED_VERSION) == false) { + yield equalTo("unsupported"); + } + yield equalTo("aggregate_metric_double"); + } + case DENSE_VECTOR -> { + logger.error("ADFDAFAF " + minVersion()); + if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_USED_TV) == false + || minVersion().supports(ESQL_DENSE_VECTOR_CREATED_VERSION) == false) { yield equalTo("unsupported"); } - yield equalTo(type.esType()); + yield equalTo("dense_vector"); } default -> equalTo(type.esType()); }; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java index 45dcfda444354..aaef3da09e2fd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java @@ -13,14 +13,28 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; +import org.elasticsearch.xpack.esql.core.type.DataType; import java.io.IOException; public class EsqlResolveFieldsResponse extends ActionResponse { - public static final TransportVersion RESOLVE_FIELDS_RESPONSE_CREATED_TV = TransportVersion.fromName( + private static final TransportVersion RESOLVE_FIELDS_RESPONSE_CREATED_TV = TransportVersion.fromName( "esql_resolve_fields_response_created" ); + /** + * Marks when we started using the minimum transport version to determine whether a data type is supported on all nodes. + * This is about the coordinator - data nodes will be able to respond with the correct data as long as they're on the + * transport version required for the respective data types. See {@link DataType#supportedVersion()}. + *

+ * Note: this is in 9.2.1, but not 9.2.0 - in 9.2.0 we resorted to workarounds to sometimes enable {@link DataType#DENSE_VECTOR} and + * {@link DataType#AGGREGATE_METRIC_DOUBLE}, even though 9.2.0 nodes already support these types. + *

+ * This means that mixed clusters with a 9.2.1 coordinator and 9.2.0 data nodes will properly support these types, + * but a 9.2.0 coordinator with 9.2.1+ nodes will still require the workaround. + */ + public static final TransportVersion RESOLVE_FIELDS_RESPONSE_USED_TV = TransportVersion.fromName("esql_resolve_fields_response_used"); + private final FieldCapabilitiesResponse caps; private final TransportVersion minTransportVersion;