diff --git a/docs/changelog/136327.yaml b/docs/changelog/136327.yaml new file mode 100644 index 0000000000000..b665e7af01a3d --- /dev/null +++ b/docs/changelog/136327.yaml @@ -0,0 +1,5 @@ +pr: 136327 +summary: Enable new data types with created version +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index f3b6079fcb68e..90786f7c0c678 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -143,19 +143,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("ml/sparse_vector_search/Search on a sparse_vector field with dots in the field names", "Vectors are no longer returned by default") task.skipTest("ml/sparse_vector_search/Search on a nested sparse_vector field with dots in the field names and conflicting child fields", "Vectors are no longer returned by default") task.skipTest("esql/190_lookup_join/lookup-no-key-only-key", "Requires the fix") - task.skipTest("esql/40_tsdb/aggregate_metric_double unsortable", "Extra function required to enable the field type") - task.skipTest("esql/40_tsdb/avg of aggregate_metric_double", "Extra function required to enable the field type") - task.skipTest("esql/40_tsdb/grouping stats on aggregate_metric_double", "Extra function required to enable the field type") - task.skipTest("esql/40_tsdb/render aggregate_metric_double when missing min and max", "Extra function required to enable the field type") - task.skipTest("esql/40_tsdb/render aggregate_metric_double when missing value", "Extra function required to enable the field type") - task.skipTest("esql/40_tsdb/sorting with aggregate_metric_double with partial submetrics", "Extra function required to enable the field type") - task.skipTest("esql/40_tsdb/stats on aggregate_metric_double missing min and max", "Extra function required to enable the field type") - task.skipTest("esql/40_tsdb/to_string aggregate_metric_double", "Extra function required to enable the field type") - task.skipTest("esql/40_tsdb/stats on aggregate_metric_double with partial submetrics", "Extra function required to enable the field type") - task.skipTest("esql/46_downsample/MV_EXPAND on non-MV aggregate metric double", "Extra function required to enable the field type") - task.skipTest("esql/46_downsample/Query stats on downsampled index", "Extra function required to enable the field type") - task.skipTest("esql/46_downsample/Render stats from downsampled index", "Extra function required to enable the field type") - task.skipTest("esql/46_downsample/Sort from multiple indices one with aggregate metric double", "Extra function required to enable the field type") }) tasks.named('yamlRestCompatTest').configure { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 756c06b72f9d8..b05d6784c072f 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.core.type; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -748,7 +749,7 @@ public DataType counter() { @Override public void writeTo(StreamOutput out) throws IOException { - if (supportedVersion.supportedOn(out.getTransportVersion()) == false) { + if (supportedVersion.supportedOn(out.getTransportVersion(), Build.current().isSnapshot()) == false) { /* * TODO when we implement version aware planning flip this to an IllegalStateException * so we throw a 500 error. It'll be our bug then. Right now it's a sign that the user diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/SupportedVersion.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/SupportedVersion.java index 6aefd1908d5b2..8881592660861 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/SupportedVersion.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/SupportedVersion.java @@ -11,15 +11,15 @@ import org.elasticsearch.TransportVersion; public interface SupportedVersion { - boolean supportedOn(TransportVersion version); + boolean supportedOn(TransportVersion version, boolean currentBuildIsSnapshot); default boolean supportedLocally() { - return supportedOn(TransportVersion.current()); + return supportedOn(TransportVersion.current(), Build.current().isSnapshot()); } SupportedVersion SUPPORTED_ON_ALL_NODES = new SupportedVersion() { @Override - public boolean supportedOn(TransportVersion version) { + public boolean supportedOn(TransportVersion version, boolean currentBuildIsSnapshot) { return true; } @@ -56,8 +56,8 @@ public String toString() { // Check usage of this constant to be sure. SupportedVersion UNDER_CONSTRUCTION = new SupportedVersion() { @Override - public boolean supportedOn(TransportVersion version) { - return Build.current().isSnapshot(); + public boolean supportedOn(TransportVersion version, boolean currentBuildIsSnapshot) { + return currentBuildIsSnapshot; } @Override @@ -76,8 +76,8 @@ public String toString() { static SupportedVersion supportedSince(TransportVersion supportedVersion) { return new SupportedVersion() { @Override - public boolean supportedOn(TransportVersion version) { - return version.supports(supportedVersion) || Build.current().isSnapshot(); + public boolean supportedOn(TransportVersion version, boolean currentBuildIsSnapshot) { + return version.supports(supportedVersion) || currentBuildIsSnapshot; } @Override diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 1e77a62711990..0617f03402730 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -43,6 +44,7 @@ import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.esql.action.EsqlResolveFieldsResponse.RESOLVE_FIELDS_RESPONSE_CREATED_TV; import static org.hamcrest.Matchers.any; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; @@ -76,11 +78,6 @@ public class AllSupportedFieldsTestCase extends ESRestTestCase { @ParametersFactory(argumentFormatting = "pref=%s mode=%s") public static List args() { - if (Build.current().isSnapshot()) { - // We only test behavior in release builds. Snapshot builds will have data types enabled that are still under construction. - return List.of(); - } - List args = new ArrayList<>(); for (MappedFieldType.FieldExtractPreference extractPreference : Arrays.asList( null, @@ -102,7 +99,7 @@ protected AllSupportedFieldsTestCase(MappedFieldType.FieldExtractPreference extr this.indexMode = indexMode; } - protected record NodeInfo(String cluster, String id, TransportVersion version, Set roles) {} + protected record NodeInfo(String cluster, String id, boolean snapshot, TransportVersion version, Set roles) {} private static Map nodeToInfo; @@ -126,6 +123,19 @@ protected boolean fetchDenseVectorAggMetricDoubleIfFns() throws IOException { return clusterHasCapability("GET", "/_query", List.of(), List.of("DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS")).orElse(false); } + private static Boolean denseVectorAggMetricDoubleIfVersion; + + private boolean denseVectorAggMetricDoubleIfVersion() throws IOException { + if (denseVectorAggMetricDoubleIfVersion == null) { + denseVectorAggMetricDoubleIfVersion = fetchDenseVectorAggMetricDoubleIfVersion(); + } + return denseVectorAggMetricDoubleIfVersion; + } + + protected boolean fetchDenseVectorAggMetricDoubleIfVersion() throws IOException { + return clusterHasCapability("GET", "/_query", List.of(), List.of("DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_VERSION")).orElse(false); + } + private static Boolean supportsNodeAssignment; protected boolean supportsNodeAssignment() throws IOException { @@ -153,11 +163,21 @@ protected static Map fetchNodeToInfo(RestClient client, String String id = (String) n.getKey(); Map nodeInfo = (Map) n.getValue(); String nodeName = (String) extractValue(nodeInfo, "name"); + + /* + * Figuring out is a node is a snapshot is kind of tricky. The main version + * doesn't include -SNAPSHOT. But ${VERSION}-SNAPSHOT is in the node info + * *somewhere*. So we do this silly toString here. + */ + String version = (String) extractValue(nodeInfo, "version"); + boolean snapshot = nodeInfo.toString().contains(version + "-SNAPSHOT"); + TransportVersion transportVersion = TransportVersion.fromId((Integer) extractValue(nodeInfo, "transport_version")); List roles = (List) nodeInfo.get("roles"); + nodeToInfo.put( nodeName, - new NodeInfo(cluster, id, transportVersion, roles.stream().map(Object::toString).collect(Collectors.toSet())) + new NodeInfo(cluster, id, snapshot, transportVersion, roles.stream().map(Object::toString).collect(Collectors.toSet())) ); } @@ -175,6 +195,22 @@ public void createIndices() throws IOException { } } + /** + * Make sure the test doesn't run on snapshot builds. Release builds only. + *

+ * {@link Build#isSnapshot()} checks if the version under test is a snapshot. + * But! This run test runs against many versions and if *any* are snapshots + * then this will fail. So we check the versions of each node in the cluster too. + *

+ */ + @Before + public void skipSnapshots() throws IOException { + assumeFalse("Only supported on production builds", Build.current().isSnapshot()); + for (NodeInfo n : allNodeToInfo().values()) { + assumeFalse("Only supported on production builds", n.snapshot()); + } + } + // TODO: Also add a test for _tsid once we can determine the minimum transport version of all nodes. public final void testFetchAll() throws IOException { Map response = esql(""" @@ -212,7 +248,7 @@ public final void testFetchAll() throws IOException { if (supportedInIndex(type) == false) { continue; } - expectedValues = expectedValues.entry(fieldName(type), expectedValue(type)); + expectedValues = expectedValues.entry(fieldName(type), expectedValue(type, nodeInfo)); } expectedValues = expectedValues.entry("_id", any(String.class)) .entry("_ignored", nullValue()) @@ -227,15 +263,23 @@ public final void testFetchAll() throws IOException { profileLogger.clearProfile(); } - // Tests a workaround and will become obsolete once we can determine the actual minimum transport version of all nodes. + /** + * Tests fetching {@code dense_vector} if possible. Uses the {@code dense_vector_agg_metric_double_if_fns} + * work around if required. + */ public final void testFetchDenseVector() throws IOException { Map response; try { - response = esql(""" - | EVAL k = v_l2_norm(f_dense_vector, [1]) // workaround to enable fetching dense_vector + String request = """ | KEEP _index, f_dense_vector | LIMIT 1000 - """); + """; + if (denseVectorAggMetricDoubleIfVersion() == false) { + request = """ + | EVAL k = v_l2_norm(f_dense_vector, [1]) // workaround to enable fetching dense_vector + """ + request; + } + response = esql(request); if ((Boolean) response.get("is_partial")) { Map clusters = (Map) response.get("_clusters"); Map details = (Map) clusters.get("details"); @@ -410,7 +454,7 @@ private void createAllTypesDoc(RestClient client, String indexName) throws IOExc } // This will become dependent on the minimum transport version of all nodes once we can determine that. - private Matcher expectedValue(DataType type) { + private Matcher expectedValue(DataType type, NodeInfo nodeInfo) throws IOException { return switch (type) { case BOOLEAN -> equalTo(true); case COUNTER_LONG, LONG, COUNTER_INTEGER, INTEGER, UNSIGNED_LONG, SHORT, BYTE -> equalTo(1); @@ -429,14 +473,24 @@ private Matcher expectedValue(DataType type) { case GEO_SHAPE -> equalTo("POINT (-71.34 41.12)"); case NULL -> nullValue(); case AGGREGATE_METRIC_DOUBLE -> { - // Currently, we cannot tell if all nodes support it or not so we treat it as unsupported. - // TODO: Fix this once we know the node versions. - yield nullValue(); + /* + * We need both AGGREGATE_METRIC_DOUBLE_CREATED and RESOLVE_FIELDS_RESPONSE_CREATED_TV + * but RESOLVE_FIELDS_RESPONSE_CREATED_TV came last so it's enough to check just it. + */ + if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_CREATED_TV) == false) { + yield nullValue(); + } + yield equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); } case DENSE_VECTOR -> { - // Currently, we cannot tell if all nodes support it or not so we treat it as unsupported. - // TODO: Fix this once we know the node versions. - yield nullValue(); + /* + * We need both DENSE_VECTOR_CREATED and RESOLVE_FIELDS_RESPONSE_CREATED_TV + * but RESOLVE_FIELDS_RESPONSE_CREATED_TV came last so it's enough to check just it. + */ + if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_CREATED_TV) == false) { + yield nullValue(); + } + yield equalTo(List.of(0.5, 10.0, 5.9999995)); } default -> throw new AssertionError("unsupported field type [" + type + "]"); }; @@ -507,7 +561,7 @@ private Map nameToValue(List names, List values) { } // This will become dependent on the minimum transport version of all nodes once we can determine that. - private Matcher expectedType(DataType type) { + private Matcher expectedType(DataType type) throws IOException { return switch (type) { case COUNTER_DOUBLE, COUNTER_LONG, COUNTER_INTEGER -> { if (indexMode == IndexMode.TIME_SERIES) { @@ -518,10 +572,16 @@ private Matcher expectedType(DataType type) { case BYTE, SHORT -> equalTo("integer"); case HALF_FLOAT, SCALED_FLOAT, FLOAT -> equalTo("double"); case NULL -> equalTo("keyword"); - // Currently unsupported without TS command or KNN function - case AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> - // TODO: Fix this once we know the node versions. - equalTo("unsupported"); + case AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> { + /* + * We need both _CREATED and RESOLVE_FIELDS_RESPONSE_CREATED_TV + * but RESOLVE_FIELDS_RESPONSE_CREATED_TV came last so it's enough to check just it. + */ + if (minVersion().supports(RESOLVE_FIELDS_RESPONSE_CREATED_TV) == false) { + yield equalTo("unsupported"); + } + yield equalTo(type.esType()); + } default -> equalTo(type.esType()); }; } @@ -555,9 +615,13 @@ private Map expectedIndices() throws IOException { name = e.getValue().cluster + ":" + name; } // We should only end up with one per cluster - result.put(name, new NodeInfo(e.getValue().cluster, null, e.getValue().version(), null)); + result.put(name, new NodeInfo(e.getValue().cluster, null, e.getValue().snapshot(), e.getValue().version(), null)); } } return result; } + + protected TransportVersion minVersion() throws IOException { + return allNodeToInfo().values().stream().map(NodeInfo::version).min(Comparator.naturalOrder()).get(); + } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec index 613b70b550789..dba1c1fbaa1f6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec @@ -1,9 +1,8 @@ retrieveBitVectorData -required_capability: dense_vector_field_type_released +required_capability: dense_vector_agg_metric_double_if_version required_capability: l2_norm_vector_similarity_function FROM dense_vector -| EVAL k = v_l2_norm(bit_vector, [1,2]) // workaround to enable fetching dense_vector | KEEP id, bit_vector | SORT id ; @@ -16,11 +15,11 @@ id:l | bit_vector:dense_vector ; denseBitVectorWithEval -required_capability: dense_vector_field_type_released +required_capability: dense_vector_agg_metric_double_if_version required_capability: l2_norm_vector_similarity_function FROM dense_vector -| EVAL v = bit_vector, k = v_l2_norm(bit_vector, [1,2]) // workaround to enable fetching dense_vector +| EVAL v = bit_vector | KEEP id, v | SORT id ; @@ -33,14 +32,13 @@ id:l | v:dense_vector ; denseBitVectorWithRenameAndDrop -required_capability: dense_vector_field_type_released +required_capability: dense_vector_agg_metric_double_if_version required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = bit_vector -| EVAL k = v_l2_norm(bit_vector, [1,2]) // workaround to enable fetching dense_vector | RENAME v AS new_vector -| DROP float_vector, byte_vector, bit_vector, k +| DROP float_vector, byte_vector, bit_vector | SORT id ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec index ac4959f58c5bc..aaa3215a88781 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec @@ -1,9 +1,8 @@ retrieveByteVectorData -required_capability: dense_vector_field_type_released +required_capability: dense_vector_agg_metric_double_if_version required_capability: l2_norm_vector_similarity_function FROM dense_vector -| EVAL k = v_l2_norm(byte_vector, [1,2,3]) // workaround to enable fetching dense_vector | KEEP id, byte_vector | SORT id ; @@ -16,12 +15,11 @@ id:l | byte_vector:dense_vector ; denseByteVectorWithEval -required_capability: dense_vector_field_type_released +required_capability: dense_vector_agg_metric_double_if_version required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = byte_vector -| EVAL k = v_l2_norm(byte_vector, [1,2,3]) // workaround to enable fetching dense_vector | KEEP id, v | SORT id ; @@ -34,14 +32,13 @@ id:l | v:dense_vector ; denseByteVectorWithRenameAndDrop -required_capability: dense_vector_field_type_released +required_capability: dense_vector_agg_metric_double_if_version required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = byte_vector -| EVAL k = v_l2_norm(byte_vector, [1,2,3]) // workaround to enable fetching dense_vector | RENAME v AS new_vector -| DROP float_vector, byte_vector, bit_vector, k +| DROP float_vector, byte_vector, bit_vector | SORT id ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec index 6a96ed14a9ed8..33a957d8a56cc 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec @@ -1,10 +1,9 @@ retrieveDenseVectorData required_capability: dense_vector_field_type_released -required_capability: dense_vector_agg_metric_double_if_fns +required_capability: dense_vector_agg_metric_double_if_version required_capability: l2_norm_vector_similarity_function FROM dense_vector -| EVAL k = v_l2_norm(float_vector, [1,2,3]) // workaround to enable fetching dense_vector | KEEP id, float_vector | SORT id ; @@ -17,12 +16,11 @@ id:l | float_vector:dense_vector ; denseVectorWithEval -required_capability: dense_vector_field_type_released +required_capability: dense_vector_agg_metric_double_if_version required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = float_vector -| EVAL k = v_l2_norm(float_vector, [1,2,3]) // workaround to enable fetching dense_vector | KEEP id, v | SORT id ; @@ -35,14 +33,13 @@ id:l | v:dense_vector ; denseVectorWithRenameAndDrop -required_capability: dense_vector_field_type_released +required_capability: dense_vector_agg_metric_double_if_version required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = float_vector -| EVAL k = v_l2_norm(float_vector, [1,2,3]) // workaround to enable fetching dense_vector | RENAME v AS new_vector -| DROP float_vector, byte_vector, bit_vector, k +| DROP float_vector, byte_vector, bit_vector | SORT id ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index 586a0d87cee71..d77ee9d6e6c59 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -4125,8 +4125,8 @@ from employees inlineStatsOnAggregateMetricDouble required_capability: inline_stats required_capability: aggregate_metric_double_v0 +required_capability: dense_vector_agg_metric_double_if_version FROM k8s-downsampled -| EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | INLINE STATS tx_max = MAX(network.eth0.tx) BY pod | SORT @timestamp, cluster, pod | KEEP @timestamp, cluster, pod, network.eth0.tx, tx_max diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java index 93f75f8395ab3..d551ca1679725 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java @@ -95,8 +95,6 @@ public void testRetrieveFieldType() { var query = """ FROM test - | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector - | DROP k """; try (var resp = run(query)) { @@ -111,7 +109,6 @@ public void testRetrieveTopNDenseVectorFieldData() { var query = """ FROM test - | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector | KEEP id, vector | SORT id ASC """; @@ -141,7 +138,6 @@ public void testRetrieveDenseVectorFieldData() { var query = """ FROM test - | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector | KEEP id, vector """; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java index f6635defd1857..af5fa7cb72db0 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java @@ -219,12 +219,10 @@ public void testKnnWithLookupJoin() { var error = expectThrows(VerificationException.class, () -> run(query)); assertThat( error.getMessage(), - // TODO revert this when we have proper versioned type resolutions - // containsString( - // "line 3:13: [KNN] function cannot operate on [lookup_vector], supplied by an index [test_lookup] in non-STANDARD " - // + "mode [lookup]" - // ) - containsString("line 3:13: Cannot use field [lookup_vector] with unsupported type [dense_vector]") + containsString( + "line 3:13: [KNN] function cannot operate on [lookup_vector], supplied by an index [test_lookup] in non-STANDARD " + + "mode [lookup]" + ) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 1338d337b1eaa..8dfc5f6a1a5b6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1479,6 +1479,8 @@ public enum Cap { DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS, + DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_VERSION, + /** * FUSE L2_NORM score normalization support */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java index 365b2b976e2f1..45dcfda444354 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java @@ -17,7 +17,7 @@ import java.io.IOException; public class EsqlResolveFieldsResponse extends ActionResponse { - private static final TransportVersion RESOLVE_FIELDS_RESPONSE_CREATED_TV = TransportVersion.fromName( + public static final TransportVersion RESOLVE_FIELDS_RESPONSE_CREATED_TV = TransportVersion.fromName( "esql_resolve_fields_response_created" ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java index baa9a9519b231..13419894ffc50 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java @@ -29,8 +29,8 @@ public record PreAnalysis( Map indexes, List enriches, List lookupIndices, - boolean supportsAggregateMetricDouble, - boolean supportsDenseVector + boolean useAggregateMetricDoubleWhenNotSupported, + boolean useDenseVectorWhenNotSupported ) { public static final PreAnalysis EMPTY = new PreAnalysis(Map.of(), List.of(), List.of(), false, false); } @@ -64,19 +64,22 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) { plan.forEachUp(Enrich.class, unresolvedEnriches::add); /* - * Enable aggregate_metric_double and dense_vector when we see certain function - * or the TS command. This allows us to release these when not all nodes understand + * Enable aggregate_metric_double and dense_vector when we see certain functions + * or the TS command. This allowed us to release these when not all nodes understand * these types. These functions are only supported on newer nodes, so we use them * as a signal that the query is only for nodes that support these types. * - * This work around is temporary until we flow the minimum transport version - * back through a cross cluster search field caps call. + * This was a workaround that was required to enable these in 9.2.0. These days + * we enable these field types if all nodes in all clusters support them. But this + * work around persists to support force-enabling them on queries that might touch + * nodes that don't have 9.2.1 or 9.3.0. If all nodes in the cluster have 9.2.1 or 9.3.0 + * this code doesn't do anything. */ - Holder supportsAggregateMetricDouble = new Holder<>(false); - Holder supportsDenseVector = new Holder<>(false); + Holder useAggregateMetricDoubleWhenNotSupported = new Holder<>(false); + Holder useDenseVectorWhenNotSupported = new Holder<>(false); indexes.forEach((ip, mode) -> { if (mode == IndexMode.TIME_SERIES) { - supportsAggregateMetricDouble.set(true); + useAggregateMetricDoubleWhenNotSupported.set(true); } }); plan.forEachDown(p -> p.forEachExpression(UnresolvedFunction.class, fn -> { @@ -88,16 +91,22 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) { || fn.name().equalsIgnoreCase("v_l2_norm") || fn.name().equalsIgnoreCase("v_dot_product") || fn.name().equalsIgnoreCase("v_magnitude")) { - supportsDenseVector.set(true); + useDenseVectorWhenNotSupported.set(true); } if (fn.name().equalsIgnoreCase("to_aggregate_metric_double")) { - supportsAggregateMetricDouble.set(true); + useAggregateMetricDoubleWhenNotSupported.set(true); } })); // mark plan as preAnalyzed (if it were marked, there would be no analysis) plan.forEachUp(LogicalPlan::setPreAnalyzed); - return new PreAnalysis(indexes, unresolvedEnriches, lookupIndices, supportsAggregateMetricDouble.get(), supportsDenseVector.get()); + return new PreAnalysis( + indexes, + unresolvedEnriches, + lookupIndices, + useAggregateMetricDoubleWhenNotSupported.get(), + useDenseVectorWhenNotSupported.get() + ); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index ed88903dd5e0f..bb92a9de1d42f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -856,8 +856,8 @@ private void preAnalyzeMainIndices( default -> requestFilter; }, indexMode == IndexMode.TIME_SERIES, - preAnalysis.supportsAggregateMetricDouble(), - preAnalysis.supportsDenseVector(), + preAnalysis.useAggregateMetricDoubleWhenNotSupported(), + preAnalysis.useDenseVectorWhenNotSupported(), listener.delegateFailureAndWrap((l, indexResolution) -> { EsqlCCSUtils.updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.inner().failures()); l.onResponse( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 49d9476126d4d..c5d89b3cb98f8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.session; +import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; @@ -16,6 +17,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.query.QueryBuilder; @@ -23,12 +25,14 @@ import org.elasticsearch.logging.Logger; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; +import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsResponse; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.DateEsField; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.KeywordEsField; +import org.elasticsearch.xpack.esql.core.type.SupportedVersion; import org.elasticsearch.xpack.esql.core.type.TextEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.index.EsIndex; @@ -45,9 +49,7 @@ import java.util.TreeMap; import java.util.TreeSet; -import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; -import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; @@ -89,8 +91,8 @@ public void resolveAsMergedMapping( Set fieldNames, QueryBuilder requestFilter, boolean includeAllDimensions, - boolean supportsAggregateMetricDouble, - boolean supportsDenseVector, + boolean useAggregateMetricDoubleWhenNotSupported, + boolean useDenseVectorWhenNotSupported, ActionListener listener ) { ActionListener> ignoreVersion = listener.delegateFailureAndWrap( @@ -102,8 +104,8 @@ public void resolveAsMergedMapping( fieldNames, requestFilter, includeAllDimensions, - supportsAggregateMetricDouble, - supportsDenseVector, + useAggregateMetricDoubleWhenNotSupported, + useDenseVectorWhenNotSupported, ignoreVersion ); } @@ -117,30 +119,81 @@ public void resolveAsMergedMappingAndRetrieveMinimumVersion( Set fieldNames, QueryBuilder requestFilter, boolean includeAllDimensions, - boolean supportsAggregateMetricDouble, - boolean supportsDenseVector, + boolean useAggregateMetricDoubleWhenNotSupported, + boolean useDenseVectorWhenNotSupported, ActionListener> listener ) { client.execute( EsqlResolveFieldsAction.TYPE, createFieldCapsRequest(indexWildcard, fieldNames, requestFilter, includeAllDimensions), listener.delegateFailureAndWrap((l, response) -> { - TransportVersion minimumVersion = response.minTransportVersion(); - - LOGGER.debug("minimum transport version {}", minimumVersion); - l.onResponse( - new Versioned<>( - mergedMappings(indexWildcard, new FieldsInfo(response.caps(), supportsAggregateMetricDouble, supportsDenseVector)), - // The minimum transport version was added to the field caps response in 9.2.1; in clusters with older nodes, - // we don't have that information and need to assume the oldest supported version. - minimumVersion == null ? TransportVersion.minimumCompatible() : minimumVersion - ) + FieldsInfo info = new FieldsInfo( + response.caps(), + response.minTransportVersion(), + Build.current().isSnapshot(), + useAggregateMetricDoubleWhenNotSupported, + useDenseVectorWhenNotSupported ); + LOGGER.debug("minimum transport version {} {}", response.minTransportVersion(), info.effectiveMinTransportVersion()); + l.onResponse(new Versioned<>(mergedMappings(indexWildcard, info), info.effectiveMinTransportVersion())); }) ); } - public record FieldsInfo(FieldCapabilitiesResponse caps, boolean supportAggregateMetricDouble, boolean supportDenseVector) {} + /** + * Information for resolving a field. + * @param caps {@link FieldCapabilitiesResponse} from all indices involved in the query + * @param minTransportVersion The minimum {@link TransportVersion} of any node that might receive the request. + * More precisely, it's the minimum transport version of ALL nodes in ALL the clusters that the query + * is targeting. It doesn't matter if the node is a data node or an ML node or a unicorn, it's transport + * version counts. BUT if the query doesn't dispatch to that cluster AT ALL, we don't count the versions + * of any nodes in that cluster. + * @param currentBuildIsSnapshot is the current build a snapshot? Note: This is always {@code Build.current().isSnapshot()} in + * production but tests need more control + * @param useAggregateMetricDoubleWhenNotSupported does the query itself force us to use {@code aggregate_metric_double} fields + * even if the remotes don't report that they support the type? This exists because + * some remotes do support {@code aggregate_metric_double} without + * reporting that they do. And, for a while, we used the query itself to opt into + * reading these fields. + * @param useDenseVectorWhenNotSupported does the query itself force us to use {@code dense_vector} fields even if the remotes don't + * report that they support the type? This exists because some remotes do + * support {@code dense_vector} without reporting that they do. And, for a while, we used the + * query itself to opt into reading these fields. + */ + public record FieldsInfo( + FieldCapabilitiesResponse caps, + @Nullable TransportVersion minTransportVersion, + boolean currentBuildIsSnapshot, + boolean useAggregateMetricDoubleWhenNotSupported, + boolean useDenseVectorWhenNotSupported + ) { + /** + * The {@link #minTransportVersion}, but if any remote didn't tell us the version we assume + * that it's very, very old. This effectively disables any fields that were created "recently". + * Which is appropriate because those fields are not supported on *almost* all versions that + * don't return the transport version in the response. + *

+ * "Very, very old" above means that there are versions of Elasticsearch that we're wire + * compatible that with that don't support sending the version back. That's anything + * from {@code 8.19.FIRST} to {@code 9.2.0}. "Recently" means any field types we + * added support for after the initial release of ESQL. These fields use + * {@link SupportedVersion#supportedOn} rather than {@link SupportedVersion#SUPPORTED_ON_ALL_NODES}. + * Except for DATE_NANOS. For DATE_NANOS we got lucky/made a mistake. It wasn't widely + * used before ESQL added support for it and we weren't careful about enabling it. So + * queries on mixed version clusters that touch DATE_NANOS will fail. All the types + * added after that, like DENSE_VECTOR, will gracefully disable themselves when talking + * to older nodes. + *

+ *

+ * Note: Once {@link EsqlResolveFieldsResponse}'s CREATED version is live everywhere + * we can remove this and make sure {@link #minTransportVersion} is non-null. That'll + * be 10.0-ish. + *

+ */ + TransportVersion effectiveMinTransportVersion() { + return minTransportVersion != null ? minTransportVersion : TransportVersion.minimumCompatible(); + } + } // public for testing only public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fieldsInfo) { @@ -275,11 +328,16 @@ private static EsField createField( IndexFieldCapabilities first = fcs.get(0); List rest = fcs.subList(1, fcs.size()); DataType type = EsqlDataTypeRegistry.INSTANCE.fromEs(first.type(), first.metricType()); - type = switch (type) { - case AGGREGATE_METRIC_DOUBLE -> fieldsInfo.supportAggregateMetricDouble ? AGGREGATE_METRIC_DOUBLE : UNSUPPORTED; - case DENSE_VECTOR -> fieldsInfo.supportDenseVector ? DENSE_VECTOR : UNSUPPORTED; - default -> type; - }; + boolean typeSupported = type.supportedVersion() + .supportedOn(fieldsInfo.effectiveMinTransportVersion(), fieldsInfo.currentBuildIsSnapshot) + || switch (type) { + case AGGREGATE_METRIC_DOUBLE -> fieldsInfo.useAggregateMetricDoubleWhenNotSupported; + case DENSE_VECTOR -> fieldsInfo.useDenseVectorWhenNotSupported; + default -> false; + }; + if (false == typeSupported) { + type = UNSUPPORTED; + } boolean aggregatable = first.isAggregatable(); EsField.TimeSeriesFieldType timeSeriesFieldType = EsField.TimeSeriesFieldType.fromIndexFieldCapabilities(first); if (rest.isEmpty() == false) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 46c8b73213abe..5a2be5973bc95 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.Build; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities; @@ -3152,16 +3153,14 @@ public void testResolveInsist_multiIndexFieldPartiallyMappedWithSingleKeywordTyp IndexResolution resolution = IndexResolver.mergedMappings( "foo,bar", - new IndexResolver.FieldsInfo( + fieldsInfoOnCurrentVersion( new FieldCapabilitiesResponse( List.of( fieldCapabilitiesIndexResponse("foo", messageResponseMap("keyword")), fieldCapabilitiesIndexResponse("bar", Map.of()) ), List.of() - ), - true, - true + ) ) ); @@ -3179,16 +3178,14 @@ public void testResolveInsist_multiIndexFieldExistsWithSingleTypeButIsNotKeyword IndexResolution resolution = IndexResolver.mergedMappings( "foo,bar", - new IndexResolver.FieldsInfo( + fieldsInfoOnCurrentVersion( new FieldCapabilitiesResponse( List.of( fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), fieldCapabilitiesIndexResponse("bar", Map.of()) ), List.of() - ), - true, - true + ) ) ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); @@ -3207,7 +3204,7 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyw IndexResolution resolution = IndexResolver.mergedMappings( "foo,bar", - new IndexResolver.FieldsInfo( + fieldsInfoOnCurrentVersion( new FieldCapabilitiesResponse( List.of( fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), @@ -3215,9 +3212,7 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyw fieldCapabilitiesIndexResponse("bazz", Map.of()) ), List.of() - ), - true, - true + ) ) ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); @@ -3235,16 +3230,14 @@ public void testResolveInsist_multiIndexSameMapping_fieldIsMapped() { IndexResolution resolution = IndexResolver.mergedMappings( "foo,bar", - new IndexResolver.FieldsInfo( + fieldsInfoOnCurrentVersion( new FieldCapabilitiesResponse( List.of( fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), fieldCapabilitiesIndexResponse("bar", messageResponseMap("long")) ), List.of() - ), - true, - true + ) ) ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); @@ -3260,7 +3253,7 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithKe IndexResolution resolution = IndexResolver.mergedMappings( "foo,bar", - new IndexResolver.FieldsInfo( + fieldsInfoOnCurrentVersion( new FieldCapabilitiesResponse( List.of( fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), @@ -3269,9 +3262,7 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithKe fieldCapabilitiesIndexResponse("qux", Map.of()) ), List.of() - ), - true, - true + ) ) ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); @@ -3289,7 +3280,7 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa IndexResolution resolution = IndexResolver.mergedMappings( "foo,bar", - new IndexResolver.FieldsInfo( + fieldsInfoOnCurrentVersion( new FieldCapabilitiesResponse( List.of( fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), @@ -3297,9 +3288,7 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa fieldCapabilitiesIndexResponse("bazz", Map.of()) ), List.of() - ), - true, - true + ) ) ); VerificationException e = expectThrows( @@ -3319,13 +3308,19 @@ public void testResolveDenseVector() { List.of() ); { - IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); + IndexResolution resolution = IndexResolver.mergedMappings( + "foo", + new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, true, true) + ); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); assertThat(plan.output().getFirst().dataType(), equalTo(DENSE_VECTOR)); } { - IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, false)); + IndexResolution resolution = IndexResolver.mergedMappings( + "foo", + new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, true, false) + ); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); assertThat(plan.output().getFirst().dataType(), equalTo(UNSUPPORTED)); @@ -3343,7 +3338,10 @@ public void testResolveAggregateMetricDouble() { List.of() ); { - IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); + IndexResolution resolution = IndexResolver.mergedMappings( + "foo", + new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, true, true) + ); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); assertThat( @@ -3352,7 +3350,10 @@ public void testResolveAggregateMetricDouble() { ); } { - IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, false, true)); + IndexResolution resolution = IndexResolver.mergedMappings( + "foo", + new IndexResolver.FieldsInfo(caps, TransportVersion.minimumCompatible(), false, false, true) + ); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); assertThat(plan.output().getFirst().dataType(), equalTo(UNSUPPORTED)); @@ -3802,7 +3803,7 @@ private static LogicalPlan analyzeWithEmptyFieldCapsResponse(String query) throw List idxResponses = List.of( new FieldCapabilitiesIndexResponse("idx", "idx", Map.of(), true, IndexMode.STANDARD) ); - IndexResolver.FieldsInfo caps = new IndexResolver.FieldsInfo(new FieldCapabilitiesResponse(idxResponses, List.of()), true, true); + IndexResolver.FieldsInfo caps = fieldsInfoOnCurrentVersion(new FieldCapabilitiesResponse(idxResponses, List.of())); IndexResolution resolution = IndexResolver.mergedMappings("test*", caps); var analyzer = analyzer(indexResolutions(resolution), TEST_VERIFIER, configuration(query)); return analyze(query, analyzer); @@ -4692,4 +4693,8 @@ static Literal string(String value) { static Literal literal(int value) { return new Literal(EMPTY, value, INTEGER); } + + static IndexResolver.FieldsInfo fieldsInfoOnCurrentVersion(FieldCapabilitiesResponse caps) { + return new IndexResolver.FieldsInfo(caps, TransportVersion.current(), false, false, false); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index fbb22c49af331..c201f544372db 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.type; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.fieldcaps.IndexFieldCapabilitiesBuilder; @@ -28,7 +29,6 @@ public void testCounter() { resolve("long", TimeSeriesParams.MetricType.COUNTER, DataType.COUNTER_LONG); resolve("integer", TimeSeriesParams.MetricType.COUNTER, DataType.COUNTER_INTEGER); resolve("double", TimeSeriesParams.MetricType.COUNTER, DataType.COUNTER_DOUBLE); - } public void testGauge() { @@ -54,7 +54,10 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); // IndexResolver uses EsqlDataTypeRegistry directly - IndexResolution resolution = IndexResolver.mergedMappings("idx-*", new IndexResolver.FieldsInfo(caps, true, true)); + IndexResolution resolution = IndexResolver.mergedMappings( + "idx-*", + new IndexResolver.FieldsInfo(caps, TransportVersion.current(), false, false, false) + ); EsField f = resolution.get().mapping().get(field); assertThat(f.getDataType(), equalTo(expected)); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 30c7d73affe19..0478548f51da1 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -231,10 +231,7 @@ filter on counter without cast: catch: bad_request esql.query: body: - query: | - FROM test - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | WHERE k8s.pod.network.tx == 1434577921 + query: 'from test | where k8s.pod.network.tx == 1434577921' --- cast counter then filter: @@ -244,8 +241,9 @@ cast counter then filter: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_v0 ] - reason: "Uses TO_AGGREGATE_METRIC_DOUBLE" + capabilities: [ dense_vector_agg_metric_double_if_version ] + reason: "uses aggregate_metric_double" + - do: esql.query: body: @@ -268,18 +266,13 @@ sort on counter without cast: - method: POST path: /_query parameters: [] - capabilities: [sorting_on_source_and_counters_forbidden, aggregate_metric_double_v0] + capabilities: [sorting_on_source_and_counters_forbidden, dense_vector_agg_metric_double_if_version] reason: "Sorting on counters shouldn't have been possible" - do: catch: /cannot sort on counter_long/ esql.query: body: - query: | - FROM test - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | KEEP k8s.pod.network.tx - | SORT k8s.pod.network.tx - | LIMIT 1 + query: 'from test | KEEP k8s.pod.network.tx | sort k8s.pod.network.tx | limit 1' --- cast then sort on counter: @@ -298,17 +291,14 @@ from doc with aggregate_metric_double: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double, dense_vector_agg_metric_double_if_version] reason: "Support for aggregate_metric_double" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | DROP a + query: 'from test2' - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} @@ -330,17 +320,14 @@ stats on aggregate_metric_double: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double, dense_vector_agg_metric_double_if_version] reason: "Support for aggregate_metric_double" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) + query: 'FROM test2 | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric)' - length: {values: 1} - length: {values.0: 4} - match: {columns.0.name: "max(agg_metric)"} @@ -364,18 +351,16 @@ grouping stats on aggregate_metric_double: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double, dense_vector_agg_metric_double_if_version] reason: "Support for aggregate_metric_double" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY dim - | SORT dim + query: "FROM test2 + | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY dim + | SORT dim" - length: {values: 2} - length: {values.0: 5} - match: {columns.0.name: "max(agg_metric)"} @@ -407,18 +392,14 @@ sorting with aggregate_metric_double with partial submetrics: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double_sorting, dense_vector_agg_metric_double_if_version] reason: "Support for sorting when aggregate_metric_double present" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test3 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | SORT @timestamp - | KEEP @timestamp, agg_metric + query: 'FROM test3 | SORT @timestamp | KEEP @timestamp, agg_metric' - length: {values: 4} - length: {values.0: 2} @@ -443,17 +424,13 @@ aggregate_metric_double unsortable: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double_sorting, dense_vector_agg_metric_double_if_version] reason: "Support for sorting when aggregate_metric_double present" - do: catch: /cannot sort on aggregate_metric_double/ esql.query: body: - query: | - FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | DROP a - | SORT agg_metric + query: 'FROM test2 | sort agg_metric' --- stats on aggregate_metric_double with partial submetrics: @@ -463,18 +440,14 @@ stats on aggregate_metric_double with partial submetrics: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double_partial_submetrics, dense_vector_agg_metric_double_if_version] reason: "Support for partial submetrics in aggregate_metric_double" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test3 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY k8s.pod.uid - | SORT k8s.pod.uid + query: 'FROM test3 | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY k8s.pod.uid | SORT k8s.pod.uid' - length: {values: 2} - length: {values.0: 5} @@ -507,17 +480,14 @@ stats on aggregate_metric_double missing min and max: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_v0 ] + capabilities: [ aggregate_metric_double_partial_submetrics, dense_vector_agg_metric_double_if_version ] reason: "Support for partial submetrics in aggregate_metric_double" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test4 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) + query: 'FROM test4 | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric)' - length: {values: 1} - length: {values.0: 4} @@ -542,17 +512,14 @@ render aggregate_metric_double when missing min and max: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_v0 ] + capabilities: [ aggregate_metric_double_rendering, dense_vector_agg_metric_double_if_version ] reason: "Support for rendering aggregate_metric_doubles" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test4 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | KEEP agg_metric + query: 'FROM test4 | KEEP agg_metric' - length: {values: 1} - length: {values.0: 1} @@ -569,18 +536,14 @@ render aggregate_metric_double when missing value: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_v0 ] + capabilities: [ aggregate_metric_double_rendering, dense_vector_agg_metric_double_if_version ] reason: "Support for rendering aggregate_metric_doubles" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test3 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | WHERE @timestamp == "2021-04-28T19:51:04.467Z" - | KEEP agg_metric + query: 'FROM test3 | WHERE @timestamp == "2021-04-28T19:51:04.467Z" | KEEP agg_metric' - length: {values: 1} - length: {values.0: 1} @@ -597,18 +560,14 @@ to_string aggregate_metric_double: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_v0 ] + capabilities: [ aggregate_metric_double_rendering, dense_vector_agg_metric_double_if_version ] reason: "Support for rendering aggregate_metric_doubles" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test4 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | EVAL agg = to_string(agg_metric) - | KEEP agg + query: 'FROM test4 | EVAL agg = to_string(agg_metric) | KEEP agg' - length: {values: 1} - length: {values.0: 1} @@ -624,17 +583,14 @@ from index pattern unsupported counter: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double_partial_submetrics, dense_vector_agg_metric_double_if_version] reason: "Support for partial submetrics in aggregate_metric_double" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test* - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | DROP a + query: 'FROM test*' - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} @@ -719,7 +675,7 @@ to_aggregate_metric_double with multi_values: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_v0 ] + capabilities: [ aggregate_metric_double_convert_to ] reason: "Support for to_aggregate_metric_double function" - do: @@ -769,7 +725,7 @@ avg of aggregate_metric_double: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double_avg, dense_vector_agg_metric_double_if_version] reason: "support avg aggregations with aggregate metric double" - do: @@ -777,11 +733,7 @@ avg of aggregate_metric_double: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: | - FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | STATS avg = avg(agg_metric) - | KEEP avg + query: 'FROM test2 | STATS avg = avg(agg_metric) | KEEP avg' - length: {values: 1} - length: {values.0: 1} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index 453d86a1bdf35..e015e73f503b2 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -145,8 +145,8 @@ unsupported: - method: POST path: /_query parameters: [] - capabilities: [dense_vector_field_type_released, dense_vector_agg_metric_double_if_fns] - reason: "uses original_type" + capabilities: [dense_vector_agg_metric_double_if_version] + reason: "fetches dense_vector" - do: allowed_warnings_regex: @@ -157,7 +157,8 @@ unsupported: query: 'from test' - match: { columns.0.name: aggregate_metric_double } - - match: { columns.0.type: unsupported } + - match: { columns.0.type: aggregate_metric_double } + - is_false: columns.0.original_types - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } - match: { columns.1.original_types: [binary] } @@ -169,7 +170,7 @@ unsupported: - match: { columns.4.name: date_range } - match: { columns.4.type: unsupported } - match: { columns.5.name: dense_vector } - - match: { columns.5.type: unsupported } + - match: { columns.5.type: dense_vector } - match: { columns.6.name: double_range } - match: { columns.6.type: unsupported } - match: { columns.7.name: float_range } @@ -218,12 +219,14 @@ unsupported: - match: { columns.28.type: integer } - length: { values: 1 } - - match: { values.0.0: null } + - match: { values.0.0: '{"min":1.0,"max":3.0,"sum":10.1,"value_count":5}' } - match: { values.0.1: null } - match: { values.0.2: null } - match: { values.0.3: "2015-01-01T12:10:30.123456789Z" } - match: { values.0.4: null } - - match: { values.0.5: null } + - match: { values.0.5.0: 0.5 } + - match: { values.0.5.1: 10.0 } + - match: { values.0.5.2: 6.0 } - match: { values.0.6: null } - match: { values.0.7: null } - match: { values.0.8: "POINT (10.0 12.0)" } @@ -255,7 +258,8 @@ unsupported: body: query: 'from test | limit 0' - match: { columns.0.name: aggregate_metric_double } - - match: { columns.0.type: unsupported } + - match: { columns.0.type: aggregate_metric_double } + - is_false: columns.0.original_types - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } - match: { columns.1.original_types: [binary] } @@ -267,7 +271,7 @@ unsupported: - match: { columns.4.name: date_range } - match: { columns.4.type: unsupported } - match: { columns.5.name: dense_vector } - - match: { columns.5.type: unsupported } + - match: { columns.5.type: dense_vector } - match: { columns.6.name: double_range } - match: { columns.6.type: unsupported } - match: { columns.7.name: float_range } @@ -338,7 +342,7 @@ unsupported with sort: - method: POST path: /_query parameters: [ ] - capabilities: [ dense_vector_field_type_released, dense_vector_agg_metric_double_if_fns ] + capabilities: [ dense_vector_agg_metric_double_if_version ] reason: "support for sorting when dense_vector_field_type present" - do: @@ -350,7 +354,7 @@ unsupported with sort: query: 'from test | sort some_doc.bar' - match: { columns.0.name: aggregate_metric_double } - - match: { columns.0.type: unsupported } + - match: { columns.0.type: aggregate_metric_double } - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } - match: { columns.2.name: completion } @@ -360,7 +364,7 @@ unsupported with sort: - match: { columns.4.name: date_range } - match: { columns.4.type: unsupported } - match: { columns.5.name: dense_vector } - - match: { columns.5.type: unsupported } + - match: { columns.5.type: dense_vector } - match: { columns.6.name: double_range } - match: { columns.6.type: unsupported } - match: { columns.7.name: float_range } @@ -409,12 +413,14 @@ unsupported with sort: - match: { columns.28.type: integer } - length: { values: 1 } - - match: { values.0.0: null } + - match: { values.0.0: '{"min":1.0,"max":3.0,"sum":10.1,"value_count":5}' } - match: { values.0.1: null } - match: { values.0.2: null } - match: { values.0.3: "2015-01-01T12:10:30.123456789Z" } - match: { values.0.4: null } - - match: { values.0.5: null } + - match: { values.0.5.0: 0.5 } + - match: { values.0.5.1: 10.0 } + - match: { values.0.5.2: 6.0 } - match: { values.0.6: null } - match: { values.0.7: null } - match: { values.0.8: "POINT (10.0 12.0)" } @@ -438,6 +444,7 @@ unsupported with sort: - match: { values.0.26: xy } - match: { values.0.27: "foo bar" } - match: { values.0.28: 3 } + --- nested declared inline: - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml index ac219b4071319..6ae4ce894d997 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml @@ -83,7 +83,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0, dense_vector_agg_metric_double_if_fns] + capabilities: [aggregate_metric_double, dense_vector_agg_metric_double_if_version] reason: "Support for aggregate_metric_double" - do: indices.downsample: @@ -98,12 +98,9 @@ setup: - do: esql.query: body: - query: | - FROM test-downsample - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | DROP a - | STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx) - | LIMIT 100 + query: "FROM test-downsample | + STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx) + | LIMIT 100" - length: {values: 1} - length: {values.0: 4} @@ -128,7 +125,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double_rendering, dense_vector_agg_metric_double_if_version] reason: "Support for rendering aggregate_metric_doubles" - do: indices.downsample: @@ -143,13 +140,7 @@ setup: - do: esql.query: body: - query: | - FROM test-downsample - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | DROP a - | WHERE @timestamp == "2021-04-28T19:00:00.000Z" - | KEEP k8s.pod.network.rx - | LIMIT 100 + query: "FROM test-downsample | WHERE @timestamp == \"2021-04-28T19:00:00.000Z\" | KEEP k8s.pod.network.rx | LIMIT 100" - length: {values: 1} - length: {values.0: 1} - match: {columns.0.name: "k8s.pod.network.rx"} @@ -164,7 +155,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double_convert_to, dense_vector_agg_metric_double_if_version] reason: "Support for to_aggregate_metric_double function" - do: @@ -240,12 +231,11 @@ setup: - do: esql.query: body: - query: | - FROM test-* - | WHERE k8s.pod.uid == "947e4ced-1786-4e53-9e0c-5c447e959507" - | EVAL rx = to_aggregate_metric_double(k8s.pod.network.rx) - | STATS max(rx), min(rx), sum(rx), count(rx) - | LIMIT 100 + query: "FROM test-* | + WHERE k8s.pod.uid == \"947e4ced-1786-4e53-9e0c-5c447e959507\" | + EVAL rx = to_aggregate_metric_double(k8s.pod.network.rx) | + STATS max(rx), min(rx), sum(rx), count(rx) | + LIMIT 100" - length: {values: 1} - length: {values.0: 4} @@ -270,7 +260,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [metrics_capability, aggregate_metric_double_implicit_casting_in_aggs, dense_vector_agg_metric_double_if_version] reason: "Support for casting aggregate metric double implicitly when present in aggregations" - do: @@ -346,12 +336,10 @@ setup: - do: esql.query: body: - query: | - FROM test-* - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | WHERE k8s.pod.uid == "947e4ced-1786-4e53-9e0c-5c447e959507" - | STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx), avg(k8s.pod.network.rx) - | LIMIT 100 + query: "FROM test-* | + WHERE k8s.pod.uid == \"947e4ced-1786-4e53-9e0c-5c447e959507\" | + STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx), avg(k8s.pod.network.rx) | + LIMIT 100" - length: {values: 1} - length: {values.0: 5} @@ -374,7 +362,7 @@ setup: - do: esql.query: body: - query: "FROM test-* | STATS max = max(k8s.pod.network.rx) | LIMIT 100" + query: "TS test-* | STATS max = max(k8s.pod.network.rx) | LIMIT 100" - length: {values: 1} - length: {values.0: 1} - match: {columns.0.name: "max"} @@ -389,7 +377,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [ts_command_v0, aggregate_metric_double_v0] + capabilities: [metrics_command, aggregate_metric_double_implicit_casting_in_aggs, dense_vector_agg_metric_double_if_version] reason: "Support for casting aggregate metric double implicitly when present in aggregations" - do: @@ -469,14 +457,12 @@ setup: - do: esql.query: body: - query: | - TS test-* - | STATS avg = sum(avg_over_time(k8s.pod.network.rx)), - count = sum(count_over_time(k8s.pod.network.rx)), - sum = sum(sum_over_time(k8s.pod.network.rx)) - BY time_bucket = bucket(@timestamp, 1 hour) - | SORT time_bucket - | LIMIT 10 + query: "TS test-* | + STATS avg = sum(avg_over_time(k8s.pod.network.rx)), + count = sum(count_over_time(k8s.pod.network.rx)), + sum = sum(sum_over_time(k8s.pod.network.rx)) + BY time_bucket = bucket(@timestamp, 1 hour) | + SORT time_bucket | LIMIT 10" - length: {values: 4} - length: {values.0: 4} @@ -513,7 +499,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [ts_command_v0, aggregate_metric_double_v0] + capabilities: [metrics_command, aggregate_metric_double_implicit_casting_in_aggs] reason: "Support for casting aggregate metric double implicitly when present in aggregations" - do: @@ -593,14 +579,13 @@ setup: - do: esql.query: body: - query: | - TS test-* - | STATS avg = sum(avg_over_time(k8s.pod.network.rx)), - count = sum(count_over_time(k8s.pod.network.rx)), - sum = sum(sum_over_time(k8s.pod.network.rx)) - BY k8s.pod.name, time_bucket = bucket(@timestamp, 1 hour) - | SORT time_bucket, k8s.pod.name - |LIMIT 10 + query: "TS test-* | + STATS avg = sum(avg_over_time(k8s.pod.network.rx)), + count = sum(count_over_time(k8s.pod.network.rx)), + sum = sum(sum_over_time(k8s.pod.network.rx)) + BY k8s.pod.name, time_bucket = bucket(@timestamp, 1 hour) | + SORT time_bucket, k8s.pod.name | + LIMIT 10" - length: {values: 6} - length: {values.0: 5} @@ -653,7 +638,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double_sorting_fixed] reason: "Fix sorting for rows comprised of docs from multiple indices where agg metric is missing from some" - do: @@ -686,13 +671,7 @@ setup: - do: esql.query: body: - query: | - FROM test-* - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | DROP a - | SORT some_field, @timestamp, k8s.pod.uid - | KEEP k8s.pod.network.rx, some_field, @timestamp - | LIMIT 10 + query: "FROM test-* | SORT some_field, @timestamp, k8s.pod.uid | KEEP k8s.pod.network.rx, some_field, @timestamp | LIMIT 10" - length: {values: 5} - length: {values.0: 3} @@ -726,7 +705,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_v0] + capabilities: [aggregate_metric_double_mv_expand, dense_vector_agg_metric_double_if_version] reason: "Have MV_EXPAND not error out when applied to aggregate_metric_doubles (is a no-op)" - do: @@ -742,14 +721,7 @@ setup: - do: esql.query: body: - query: | - FROM test-downsample - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double - | DROP a - | MV_EXPAND k8s.pod.network.rx - | SORT @timestamp, k8s.pod.uid - | KEEP k8s.pod.network.rx, @timestamp - | LIMIT 10 + query: "FROM test-downsample | MV_EXPAND k8s.pod.network.rx | SORT @timestamp, k8s.pod.uid | KEEP k8s.pod.network.rx, @timestamp | LIMIT 10" - length: {values: 4} - length: {values.0: 2}