From 870db5f520a03cdca2d4e33bd71ed87650cfd89d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 22 Sep 2025 14:20:13 -0400 Subject: [PATCH 01/38] ESQL: Test fetching old types --- .../esql/qa/server/mixed-cluster/build.gradle | 4 + .../esql/qa/mixed/AllSupportedFieldsIT.java | 31 ++ .../qa/multi_node/AllSupportedFieldsIT.java | 31 ++ .../qa/single_node/AllSupportedFieldsIT.java | 32 ++ .../qa/rest/AllSupportedFieldsTestCase.java | 388 ++++++++++++++++++ .../esql/qa/rest/FieldExtractorTestCase.java | 2 +- 6 files changed, 487 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java create mode 100644 x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java create mode 100644 x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AllSupportedFieldsIT.java create mode 100644 x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle index 4f88d4303f108..bae8f1be07847 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle @@ -35,6 +35,10 @@ dependencies { javaRestTestImplementation project(xpackModule('esql:qa:testFixtures')) javaRestTestImplementation project(xpackModule('esql:qa:server')) javaRestTestImplementation project(xpackModule('esql')) + + clusterPlugins project(':plugins:mapper-size') + clusterPlugins project(':plugins:mapper-murmur3') + clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin') } GradleUtils.extendSourceSet(project, "javaRestTest", "yamlRestTest") diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java new file mode 100644 index 0000000000000..ab090457421db --- /dev/null +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.mixed; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xpack.esql.qa.rest.AllSupportedFieldsTestCase; +import org.junit.ClassRule; + +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class AllSupportedFieldsIT extends AllSupportedFieldsTestCase { + @ClassRule + public static ElasticsearchCluster cluster = Clusters.mixedVersionCluster(); + + public AllSupportedFieldsIT(MappedFieldType.FieldExtractPreference extractPreference, IndexMode indexMode) { + super(extractPreference, indexMode); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java new file mode 100644 index 0000000000000..b9054e1c1a360 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.multi_node; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xpack.esql.qa.rest.AllSupportedFieldsTestCase; +import org.junit.ClassRule; + +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class AllSupportedFieldsIT extends AllSupportedFieldsTestCase { + @ClassRule + public static ElasticsearchCluster cluster = Clusters.testCluster(c -> {}); + + public AllSupportedFieldsIT(MappedFieldType.FieldExtractPreference extractPreference, IndexMode indexMode) { + super(extractPreference, indexMode); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AllSupportedFieldsIT.java new file mode 100644 index 0000000000000..dddc806d001f8 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AllSupportedFieldsIT.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.single_node; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xpack.esql.qa.rest.AllSupportedFieldsTestCase; +import org.junit.ClassRule; + +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class AllSupportedFieldsIT extends AllSupportedFieldsTestCase { + @ClassRule + public static ElasticsearchCluster cluster = Clusters.testCluster(c -> {}); + + public AllSupportedFieldsIT(MappedFieldType.FieldExtractPreference extractPreference, IndexMode indexMode) { + super(extractPreference, indexMode); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java new file mode 100644 index 0000000000000..1cb859ee73b56 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -0,0 +1,388 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.rest; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.test.MapMatcher; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.hamcrest.Matcher; +import org.junit.Before; +import org.junit.Rule; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.hamcrest.Matchers.any; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +/** + * Creates indices with all supported fields and fetches values from them. + *

+ * In a single cluster where all nodes are on a single version this is + * just an "is it plugged in" style smoke test. In a mixed version cluster + * this is testing the behavior of fetching potentially unsupported field + * types. The same is true multi-cluster cases. + *

+ *

+ * This isn't trying to test complex interactions with field loading so we + * load constant field values and have simple mappings. + *

+ */ +public class AllSupportedFieldsTestCase extends ESRestTestCase { + private static final Logger logger = LogManager.getLogger(FieldExtractorTestCase.class); + + @Rule(order = Integer.MIN_VALUE) + public ProfileLogger profileLogger = new ProfileLogger(); + + @ParametersFactory(argumentFormatting = "pref=%s mode=%s") + public static List args() { + List args = new ArrayList<>(); + for (MappedFieldType.FieldExtractPreference extractPreference : Arrays.asList( + null, + MappedFieldType.FieldExtractPreference.NONE, + MappedFieldType.FieldExtractPreference.STORED + )) { + for (IndexMode indexMode : IndexMode.values()) { + args.add(new Object[] { extractPreference, indexMode }); + } + } + return args; + } + + private final MappedFieldType.FieldExtractPreference extractPreference; + private final IndexMode indexMode; + + private record NodeInfo(String id, Version version) {} + private Map nodeToInfo; + + protected AllSupportedFieldsTestCase(MappedFieldType.FieldExtractPreference extractPreference, IndexMode indexMode) { + this.extractPreference = extractPreference; + this.indexMode = indexMode; + } + + private Map nodeToInfo() throws IOException { + if (nodeToInfo != null) { + return nodeToInfo; + } + + nodeToInfo = new TreeMap<>(); + Request getIds = new Request("GET", "_cat/nodes"); + getIds.addParameter("h", "name,id,version"); + getIds.addParameter("full_id", ""); + Response idsResponse = client().performRequest(getIds); + BufferedReader idsReader = new BufferedReader(new InputStreamReader(idsResponse.getEntity().getContent())); + String line; + while ((line = idsReader.readLine()) != null) { + String[] l = line.split(" "); + // TODO what's the right thing to use instead of Version? + nodeToInfo.put(l[0], new NodeInfo(l[1], Version.fromString(l[2]))); + } + return nodeToInfo; + } + + @Before + public void createIndices() throws IOException { + for (Map.Entry e : nodeToInfo().entrySet()) { + createIndexForNode(e.getKey(), e.getValue().id()); + } + } + + public final void test() throws IOException { + Request request = new Request("POST", "_query"); + XContentBuilder body = JsonXContent.contentBuilder().startObject(); + body.field("query", """ + FROM %mode%* METADATA _id, _ignored, _index, _index_mode, _score, _source, _version + | LIMIT 1000 + """.replace("%mode%", indexMode.toString())); + { + body.startObject("pragma"); + if (extractPreference != null) { + body.field("field_extract_preference", extractPreference); + } + body.endObject(); + } + body.field("accept_pragma_risks", "true"); + body.field("profile", true); + body.endObject(); + request.setJsonEntity(Strings.toString(body)); + + Map response = responseAsMap(client().performRequest(request)); + List columns = (List) response.get("columns"); + List values = (List) response.get("values"); + profileLogger.extractProfile(response, true); + + MapMatcher expectedColumns = matchesMap(); + for (DataType type : DataType.values()) { + if (supportedInIndex(type) == false) { + continue; + } + expectedColumns = expectedColumns.entry(fieldName(type), expectedType(type)); + } + expectedColumns = expectedColumns.entry("_id", "keyword") + .entry("_ignored", "keyword") + .entry("_index", "keyword") + .entry("_index_mode", "keyword") + .entry("_score", "double") + .entry("_source", "_source") + .entry("_version", "long"); + assertMap(nameToType(columns), expectedColumns); + + MapMatcher expectedAllValues = matchesMap(); + for (Map.Entry e : nodeToInfo().entrySet()) { + String nodeName = e.getKey(); + NodeInfo nodeInfo = e.getValue(); + MapMatcher expectedValues = matchesMap(); + for (DataType type : DataType.values()) { + if (supportedInIndex(type) == false) { + continue; + } + expectedValues = expectedValues.entry(fieldName(type), expectedValue(nodeInfo.version, type)); + } + String expectedIndex = indexMode + "_" + nodeName; + expectedValues = expectedValues.entry("_id", any(String.class)) + .entry("_ignored", nullValue()) + .entry("_index", expectedIndex) + .entry("_index_mode", indexMode.toString()) + .entry("_score", 0.0) + .entry("_source", matchesMap().extraOk()) + .entry("_version", 1); + expectedAllValues = expectedAllValues.entry(expectedIndex, expectedValues); + } + assertMap(indexToRow(columns, values), expectedAllValues); + profileLogger.clearProfile(); + } + + private void createIndexForNode(String nodeName, String nodeId) throws IOException { + String indexName = indexMode + "_" + nodeName; + if (false == indexExists(indexName)) { + createAllTypesIndex(indexName, nodeId); + createAllTypesDoc(indexName); + } + } + + private void createAllTypesIndex(String indexName, String nodeId) throws IOException { + XContentBuilder config = JsonXContent.contentBuilder().startObject(); + { + config.startObject("settings"); + config.startObject("index"); + config.field("mode", indexMode); + if (indexMode == IndexMode.TIME_SERIES) { + config.field("routing_path", "f_keyword"); + } + config.field("routing.allocation.include._id", nodeId); + config.endObject(); + config.endObject(); + } + { + config.startObject("mappings").startObject("properties"); + for (DataType type : DataType.values()) { + if (supportedInIndex(type) == false) { + continue; + } + config.startObject(fieldName(type)); + typeMapping(config, type); + config.endObject(); + } + config.endObject().endObject().endObject(); + } + Request request = new Request("PUT", indexName); + request.setJsonEntity(Strings.toString(config)); + client().performRequest(request); + } + + private String fieldName(DataType type) { + return type == DataType.DATETIME ? "@timestamp" : "f_" + type.esType(); + } + + private void typeMapping(XContentBuilder config, DataType type) throws IOException { + switch (type) { + case COUNTER_DOUBLE, COUNTER_INTEGER, COUNTER_LONG -> config.field("type", type.esType().replace("counter_", "")) + .field("time_series_metric", "counter"); + case SCALED_FLOAT -> config.field("type", type.esType()).field("scaling_factor", 1); + case AGGREGATE_METRIC_DOUBLE -> config.field("type", type.esType()) + .field("metrics", List.of("min", "max", "sum", "value_count")) + .field("default_metric", "max"); + case NULL -> config.field("type", "keyword"); + case KEYWORD -> { + config.field("type", type.esType()); + if (indexMode == IndexMode.TIME_SERIES) { + config.field("time_series_dimension", true); + } + } + default -> config.field("type", type.esType()); + } + } + + private void createAllTypesDoc(String indexName) throws IOException { + XContentBuilder doc = JsonXContent.contentBuilder().startObject(); + for (DataType type : DataType.values()) { + if (supportedInIndex(type) == false) { + continue; + } + doc.field(fieldName(type)); + switch (type) { + case BOOLEAN -> doc.value(true); + case COUNTER_LONG, LONG, COUNTER_INTEGER, INTEGER, UNSIGNED_LONG, SHORT, BYTE -> doc.value(1); + case COUNTER_DOUBLE, DOUBLE, FLOAT, HALF_FLOAT, SCALED_FLOAT -> doc.value(1.1); + case KEYWORD, TEXT -> doc.value("foo"); + case DATETIME, DATE_NANOS -> doc.value("2025-01-01T01:00:00Z"); + case IP -> doc.value("192.168.0.1"); + case VERSION -> doc.value("1.0.0-SNAPSHOT"); + case GEO_POINT, GEO_SHAPE -> doc.value("POINT (-71.34 41.12)"); + case NULL -> doc.nullValue(); + case AGGREGATE_METRIC_DOUBLE -> { + doc.startObject(); + doc.field("min", -302.50); + doc.field("max", 702.30); + doc.field("sum", 200.0); + doc.field("value_count", 25); + doc.endObject(); + } + case DENSE_VECTOR -> doc.value(List.of(0.5, 10, 6)); + default -> throw new AssertionError("unsupported field type [" + type + "]"); + } + } + doc.endObject(); + Request request = new Request("POST", indexName + "/_doc"); + request.addParameter("refresh", ""); + request.setJsonEntity(Strings.toString(doc)); + client().performRequest(request); + } + + private Matcher expectedValue(Version version, DataType type) { + return switch (type) { + case BOOLEAN -> equalTo(true); + case COUNTER_LONG, LONG, COUNTER_INTEGER, INTEGER, UNSIGNED_LONG, SHORT, BYTE -> equalTo(1); + case COUNTER_DOUBLE, DOUBLE -> equalTo(1.1); + case FLOAT -> equalTo(1.100000023841858); + case HALF_FLOAT -> equalTo(1.099609375); + case SCALED_FLOAT -> equalTo(1.0); + // TODO what about the extra types and ES supports and ESQL flattens away like semantic_text and wildcard? + case KEYWORD, TEXT -> equalTo("foo"); + case DATETIME, DATE_NANOS -> equalTo("2025-01-01T01:00:00.000Z"); + case IP -> equalTo("192.168.0.1"); + case VERSION -> equalTo("1.0.0-SNAPSHOT"); + case GEO_POINT -> extractPreference == MappedFieldType.FieldExtractPreference.DOC_VALUES || syntheticSourceByDefault() + ? equalTo("POINT (-71.34000004269183 41.1199999647215)") + : equalTo("POINT (-71.34 41.12)"); + case GEO_SHAPE -> equalTo("POINT (-71.34 41.12)"); + case NULL -> nullValue(); + case AGGREGATE_METRIC_DOUBLE -> + // TODO why not a map? + equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); + case DENSE_VECTOR -> + version.onOrAfter(Version.V_9_2_0) + ? matchesList().item(0.5).item(10.0).item(5.9999995) + : matchesList().item(0.04283529).item(0.85670584).item(0.5140235); + default -> throw new AssertionError("unsupported field type [" + type + "]"); + }; + } + + /** + * Is the type supported in indices? + */ + private boolean supportedInIndex(DataType t) { + return switch (t) { + // These are supported but implied by the index process. + case OBJECT, SOURCE, DOC_DATA_TYPE, TSID_DATA_TYPE, + // Internal only + UNSUPPORTED, PARTIAL_AGG, + // You can't index these - they are just constants. + DATE_PERIOD, TIME_DURATION, GEOTILE, GEOHASH, GEOHEX, + // TODO fix geo + CARTESIAN_POINT, CARTESIAN_SHAPE -> false; + default -> true; + }; + } + + private Map nameToType(List columns) { + Map result = new TreeMap<>(); + for (Object c : columns) { + Map map = (Map) c; + result.put(map.get("name").toString(), map.get("type")); + } + return result; + } + + private List names(List columns) { + List result = new ArrayList<>(); + for (Object c : columns) { + Map map = (Map) c; + result.add(map.get("name").toString()); + } + return result; + } + + private Map> indexToRow(List columns, List values) { + List names = names(columns); + int timestampIdx = names.indexOf("_index"); + Map> result = new TreeMap<>(); + for (Object r : values) { + List row = (List) r; + result.put(row.get(timestampIdx).toString(), nameToValue(names, row)); + } + return result; + } + + private Map nameToValue(List names, List values) { + Map result = new TreeMap<>(); + for (int i = 0; i < values.size(); i++) { + result.put(names.get(i), values.get(i)); + } + return result; + } + + private String expectedType(DataType type) { + return switch (type) { + case COUNTER_DOUBLE, COUNTER_LONG, COUNTER_INTEGER -> { + if (indexMode == IndexMode.TIME_SERIES) { + yield type.esType(); + } + yield type.esType().replace("counter_", ""); + } + case BYTE, SHORT -> "integer"; + case HALF_FLOAT, SCALED_FLOAT, FLOAT -> "double"; + case NULL -> "keyword"; + default -> type.esType(); + }; + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + private boolean syntheticSourceByDefault() { + return switch (indexMode) { + case TIME_SERIES, LOGSDB -> true; + case STANDARD, LOOKUP -> false; + }; + } +} diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java index 1eb2b33b925e0..7d502e87c5f81 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java @@ -73,7 +73,7 @@ public abstract class FieldExtractorTestCase extends ESRestTestCase { public ProfileLogger profileLogger = new ProfileLogger(); @ParametersFactory(argumentFormatting = "%s") - public static List args() throws Exception { + public static List args() { return List.of( new Object[] { null }, new Object[] { MappedFieldType.FieldExtractPreference.NONE }, From c541fbd95326174205f248c6cd6963c91370d5ac Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 22 Sep 2025 18:27:58 +0000 Subject: [PATCH 02/38] [CI] Auto commit changes from spotless --- .../xpack/esql/qa/mixed/AllSupportedFieldsIT.java | 1 + .../xpack/esql/qa/multi_node/AllSupportedFieldsIT.java | 1 + .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java index ab090457421db..8901f8aa70ea5 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.qa.mixed; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.test.TestClustersThreadFilter; diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java index b9054e1c1a360..cdc9e51d84262 100644 --- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java +++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.qa.multi_node; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.test.TestClustersThreadFilter; diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 1cb859ee73b56..e6245c7fbe19f 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -80,6 +80,7 @@ public static List args() { private final IndexMode indexMode; private record NodeInfo(String id, Version version) {} + private Map nodeToInfo; protected AllSupportedFieldsTestCase(MappedFieldType.FieldExtractPreference extractPreference, IndexMode indexMode) { @@ -297,8 +298,7 @@ private Matcher expectedValue(Version version, DataType type) { case AGGREGATE_METRIC_DOUBLE -> // TODO why not a map? equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); - case DENSE_VECTOR -> - version.onOrAfter(Version.V_9_2_0) + case DENSE_VECTOR -> version.onOrAfter(Version.V_9_2_0) ? matchesList().item(0.5).item(10.0).item(5.9999995) : matchesList().item(0.04283529).item(0.85670584).item(0.5140235); default -> throw new AssertionError("unsupported field type [" + type + "]"); From ef1da90a1f50816c11bfb2f056908499a709ace7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 22 Sep 2025 14:37:00 -0400 Subject: [PATCH 03/38] Drop unused --- x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle index bae8f1be07847..4f88d4303f108 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle @@ -35,10 +35,6 @@ dependencies { javaRestTestImplementation project(xpackModule('esql:qa:testFixtures')) javaRestTestImplementation project(xpackModule('esql:qa:server')) javaRestTestImplementation project(xpackModule('esql')) - - clusterPlugins project(':plugins:mapper-size') - clusterPlugins project(':plugins:mapper-murmur3') - clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin') } GradleUtils.extendSourceSet(project, "javaRestTest", "yamlRestTest") From c5c40cb26264875c8e3864ff51d0c038cf5d83ab Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 22 Sep 2025 19:18:11 +0000 Subject: [PATCH 04/38] [CI] Update transport version definitions --- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index bf1a90e5be4e9..6e7d51d3d3020 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -index_request_include_tsid,9167000 +security_stats_endpoint,9168000 From ce91be11cf8c568107265400fde1320c75994aa6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 22 Sep 2025 21:12:54 -0400 Subject: [PATCH 05/38] better logging --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index e6245c7fbe19f..78b6095fa1ae3 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -101,6 +101,7 @@ private Map nodeToInfo() throws IOException { BufferedReader idsReader = new BufferedReader(new InputStreamReader(idsResponse.getEntity().getContent())); String line; while ((line = idsReader.readLine()) != null) { + logger.info("node: {}", line); String[] l = line.split(" "); // TODO what's the right thing to use instead of Version? nodeToInfo.put(l[0], new NodeInfo(l[1], Version.fromString(l[2]))); @@ -135,10 +136,15 @@ public final void test() throws IOException { request.setJsonEntity(Strings.toString(body)); Map response = responseAsMap(client().performRequest(request)); + if ((Boolean) response.get("is_partial")) { + throw new AssertionError("partial results: " + response); + } + List columns = (List) response.get("columns"); List values = (List) response.get("values"); profileLogger.extractProfile(response, true); + MapMatcher expectedColumns = matchesMap(); for (DataType type : DataType.values()) { if (supportedInIndex(type) == false) { From 654abdedbbb5205422fa37a736a9321639e74522 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 23 Sep 2025 01:20:45 +0000 Subject: [PATCH 06/38] [CI] Auto commit changes from spotless --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 78b6095fa1ae3..55ef5a719bcd9 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -144,7 +144,6 @@ public final void test() throws IOException { List values = (List) response.get("values"); profileLogger.extractProfile(response, true); - MapMatcher expectedColumns = matchesMap(); for (DataType type : DataType.values()) { if (supportedInIndex(type) == false) { From 1945205c6edc18ec1720baa34197b38586d7a5bb Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 22 Sep 2025 21:52:41 -0400 Subject: [PATCH 07/38] Fix? --- .../xpack/esql/core/type/DataType.java | 47 +++++++++++++++++-- .../qa/rest/AllSupportedFieldsTestCase.java | 19 +++++--- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index b3b36ac0fa160..0b37f26357159 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -7,8 +7,10 @@ package org.elasticsearch.xpack.esql.core.type; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -32,6 +34,9 @@ import java.util.function.Function; import static java.util.stream.Collectors.toMap; +import static org.elasticsearch.TransportVersions.INDEXING_PRESSURE_THROTTLING_STATS; +import static org.elasticsearch.TransportVersions.INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING; +import static org.elasticsearch.TransportVersions.ML_INFERENCE_SAGEMAKER_CHAT_COMPLETION; /** * This enum represents data types the ES|QL query processing layer is able to @@ -140,7 +145,7 @@ * unsupported types. * */ -public enum DataType { +public enum DataType implements Writeable { /** * Fields of this type are unsupported by any functions and are always * rendered as {@code null} in the response. @@ -306,12 +311,26 @@ public enum DataType { */ PARTIAL_AGG(builder().esType("partial_agg").estimatedSize(1024)), - AGGREGATE_METRIC_DOUBLE(builder().esType("aggregate_metric_double").estimatedSize(Double.BYTES * 3 + Integer.BYTES)), + AGGREGATE_METRIC_DOUBLE( + builder().esType("aggregate_metric_double") + .estimatedSize(Double.BYTES * 3 + Integer.BYTES) + .createdVersion( + // Version created just *after* we committed support for aggregate_metric_double + INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING + ) + ), /** * Fields with this type are dense vectors, represented as an array of double values. */ - DENSE_VECTOR(builder().esType("dense_vector").estimatedSize(4096)); + DENSE_VECTOR( + builder().esType("dense_vector") + .estimatedSize(4096) + .createdVersion( + // Version created just *after* we committed support for dense_vector + ML_INFERENCE_SAGEMAKER_CHAT_COMPLETION + ) + ); /** * Types that are actively being built. These types are @@ -375,6 +394,11 @@ public enum DataType { */ private final DataType counter; + /** + * Version that first created this data type. + */ + private final TransportVersion createdVersion; + DataType(Builder builder) { String typeString = builder.typeName != null ? builder.typeName : builder.esType; this.typeName = typeString.toLowerCase(Locale.ROOT); @@ -387,6 +411,7 @@ public enum DataType { this.isCounter = builder.isCounter; this.widenSmallNumeric = builder.widenSmallNumeric; this.counter = builder.counter; + this.createdVersion = builder.createdVersion; } private static final Collection TYPES = Arrays.stream(values()) @@ -727,8 +752,10 @@ public DataType counter() { return counter; } + @Override public void writeTo(StreamOutput out) throws IOException { - ((PlanStreamOutput) out).writeCachedString(typeName); + String name = out.getTransportVersion().supports(createdVersion) ? typeName : UNSUPPORTED.typeName; + ((PlanStreamOutput) out).writeCachedString(name); } public static DataType readFrom(StreamInput in) throws IOException { @@ -846,6 +873,13 @@ private static class Builder { */ private DataType counter; + /** + * The version when this data type was created. We default to the first + * version for which we maintain wire compatibility, which is pretty + * much {@code 8.18.0}. + */ + private TransportVersion createdVersion = INDEXING_PRESSURE_THROTTLING_STATS; + Builder() {} Builder esType(String esType) { @@ -901,5 +935,10 @@ Builder counter(DataType counter) { this.counter = counter; return this; } + + Builder createdVersion(TransportVersion createdVersion) { + this.createdVersion = createdVersion; + return this; + } } } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 78b6095fa1ae3..939154f67c740 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -144,7 +144,6 @@ public final void test() throws IOException { List values = (List) response.get("values"); profileLogger.extractProfile(response, true); - MapMatcher expectedColumns = matchesMap(); for (DataType type : DataType.values()) { if (supportedInIndex(type) == false) { @@ -301,12 +300,20 @@ private Matcher expectedValue(Version version, DataType type) { : equalTo("POINT (-71.34 41.12)"); case GEO_SHAPE -> equalTo("POINT (-71.34 41.12)"); case NULL -> nullValue(); - case AGGREGATE_METRIC_DOUBLE -> + case AGGREGATE_METRIC_DOUBLE -> { + // TODO this is almost certainly not the right version + if (false == version.onOrAfter(Version.V_9_0_4)) { + yield nullValue(); + } // TODO why not a map? - equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); - case DENSE_VECTOR -> version.onOrAfter(Version.V_9_2_0) - ? matchesList().item(0.5).item(10.0).item(5.9999995) - : matchesList().item(0.04283529).item(0.85670584).item(0.5140235); + yield equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); + } + case DENSE_VECTOR -> { + if (false == version.onOrAfter(Version.V_9_2_0)) { + yield matchesList().item(0.04283529).item(0.85670584).item(0.5140235); + } + yield matchesList().item(0.5).item(10.0).item(5.9999995); + } default -> throw new AssertionError("unsupported field type [" + type + "]"); }; } From 67b4e73477bb493ddd126b2fa7b52dfa63ec10aa Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 23 Sep 2025 12:17:31 -0400 Subject: [PATCH 08/38] Multi And change bwc --- .../xpack/esql/core/type/DataType.java | 8 +- .../xpack/esql/ccq/AllSupportedFieldsIT.java | 88 +++++++++++++++++++ .../qa/multi_node/AllSupportedFieldsIT.java | 32 ------- .../qa/rest/AllSupportedFieldsTestCase.java | 53 ++++++----- .../xpack/esql/analysis/Analyzer.java | 1 + 5 files changed, 127 insertions(+), 55 deletions(-) create mode 100644 x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java delete mode 100644 x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 0b37f26357159..e837d928dac78 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -754,8 +754,12 @@ public DataType counter() { @Override public void writeTo(StreamOutput out) throws IOException { - String name = out.getTransportVersion().supports(createdVersion) ? typeName : UNSUPPORTED.typeName; - ((PlanStreamOutput) out).writeCachedString(name); + if (out.getTransportVersion().supports(createdVersion) == false) { + throw new IllegalStateException( + "remote node at version [" + out.getTransportVersion() + "] doesn't understand data type [" + this + "]" + ); + } + ((PlanStreamOutput) out).writeCachedString(typeName); } public static DataType readFrom(StreamInput in) throws IOException { diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java new file mode 100644 index 0000000000000..0ce60186cf05a --- /dev/null +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.ccq; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xpack.esql.qa.rest.AllSupportedFieldsTestCase; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.Map; +import java.util.TreeMap; + +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class AllSupportedFieldsIT extends AllSupportedFieldsTestCase { + static ElasticsearchCluster remoteCluster = Clusters.remoteCluster(); + static ElasticsearchCluster localCluster = Clusters.localCluster(remoteCluster); + + @ClassRule + public static TestRule clusterRule = RuleChain.outerRule(remoteCluster).around(localCluster); + + private static RestClient remoteClient; + private static Map remoteNodeToInfo; + + public AllSupportedFieldsIT(MappedFieldType.FieldExtractPreference extractPreference, IndexMode indexMode) { + super(extractPreference, indexMode); + } + + @Before + public void createRemoteIndices() throws IOException { + for (Map.Entry e : remoteNodeToInfo().entrySet()) { + createIndexForNode(remoteClient(), e.getKey(), e.getValue().id()); + } + } + + private Map remoteNodeToInfo() throws IOException { + if (remoteNodeToInfo == null) { + remoteNodeToInfo = fetchNodeToInfo(remoteClient()); + } + return remoteNodeToInfo; + } + + @Override + protected Map allNodeToInfo() throws IOException { + Map all = new TreeMap<>(); + all.putAll(super.allNodeToInfo()); + all.putAll(remoteNodeToInfo()); + return all; + } + + private RestClient remoteClient() throws IOException { + if (remoteClient == null) { + var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses()); + remoteClient = buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0])); + } + return remoteClient; + } + + @Override + protected String getTestRestCluster() { + return localCluster.getHttpAddresses(); + } + + @AfterClass + public static void closeRemoteClient() throws IOException { + try { + IOUtils.close(remoteClient); + } finally { + remoteClient = null; + } + } +} diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java deleted file mode 100644 index cdc9e51d84262..0000000000000 --- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/AllSupportedFieldsIT.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.qa.multi_node; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; - -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.test.TestClustersThreadFilter; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.xpack.esql.qa.rest.AllSupportedFieldsTestCase; -import org.junit.ClassRule; - -@ThreadLeakFilters(filters = TestClustersThreadFilter.class) -public class AllSupportedFieldsIT extends AllSupportedFieldsTestCase { - @ClassRule - public static ElasticsearchCluster cluster = Clusters.testCluster(c -> {}); - - public AllSupportedFieldsIT(MappedFieldType.FieldExtractPreference extractPreference, IndexMode indexMode) { - super(extractPreference, indexMode); - } - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } -} diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 939154f67c740..1d0f29f594c43 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -12,6 +12,7 @@ import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.MappedFieldType; @@ -79,25 +80,34 @@ public static List args() { private final MappedFieldType.FieldExtractPreference extractPreference; private final IndexMode indexMode; - private record NodeInfo(String id, Version version) {} - - private Map nodeToInfo; - protected AllSupportedFieldsTestCase(MappedFieldType.FieldExtractPreference extractPreference, IndexMode indexMode) { this.extractPreference = extractPreference; this.indexMode = indexMode; } + protected record NodeInfo(String id, Version version) {} + + private static Map nodeToInfo; private Map nodeToInfo() throws IOException { - if (nodeToInfo != null) { - return nodeToInfo; + if (nodeToInfo == null) { + nodeToInfo = fetchNodeToInfo(client()); } + return nodeToInfo; + } - nodeToInfo = new TreeMap<>(); + /** + * Map from node name to information about the node. + */ + protected Map allNodeToInfo() throws IOException { + return nodeToInfo(); + } + + protected static Map fetchNodeToInfo(RestClient client) throws IOException { + Map nodeToInfo = new TreeMap<>(); Request getIds = new Request("GET", "_cat/nodes"); getIds.addParameter("h", "name,id,version"); getIds.addParameter("full_id", ""); - Response idsResponse = client().performRequest(getIds); + Response idsResponse = client.performRequest(getIds); BufferedReader idsReader = new BufferedReader(new InputStreamReader(idsResponse.getEntity().getContent())); String line; while ((line = idsReader.readLine()) != null) { @@ -112,7 +122,7 @@ private Map nodeToInfo() throws IOException { @Before public void createIndices() throws IOException { for (Map.Entry e : nodeToInfo().entrySet()) { - createIndexForNode(e.getKey(), e.getValue().id()); + createIndexForNode(client(), e.getKey(), e.getValue().id()); } } @@ -132,6 +142,7 @@ public final void test() throws IOException { } body.field("accept_pragma_risks", "true"); body.field("profile", true); + body.field("include_ccs_metadata", true); body.endObject(); request.setJsonEntity(Strings.toString(body)); @@ -161,7 +172,7 @@ public final void test() throws IOException { assertMap(nameToType(columns), expectedColumns); MapMatcher expectedAllValues = matchesMap(); - for (Map.Entry e : nodeToInfo().entrySet()) { + for (Map.Entry e : allNodeToInfo().entrySet()) { String nodeName = e.getKey(); NodeInfo nodeInfo = e.getValue(); MapMatcher expectedValues = matchesMap(); @@ -185,15 +196,15 @@ public final void test() throws IOException { profileLogger.clearProfile(); } - private void createIndexForNode(String nodeName, String nodeId) throws IOException { + protected void createIndexForNode(RestClient client, String nodeName, String nodeId) throws IOException { String indexName = indexMode + "_" + nodeName; - if (false == indexExists(indexName)) { - createAllTypesIndex(indexName, nodeId); - createAllTypesDoc(indexName); + if (false == indexExists(client, indexName)) { + createAllTypesIndex(client, indexName, nodeId); + createAllTypesDoc(client, indexName); } } - private void createAllTypesIndex(String indexName, String nodeId) throws IOException { + private void createAllTypesIndex(RestClient client, String indexName, String nodeId) throws IOException { XContentBuilder config = JsonXContent.contentBuilder().startObject(); { config.startObject("settings"); @@ -213,21 +224,21 @@ private void createAllTypesIndex(String indexName, String nodeId) throws IOExcep continue; } config.startObject(fieldName(type)); - typeMapping(config, type); + typeMapping(indexMode, config, type); config.endObject(); } config.endObject().endObject().endObject(); } Request request = new Request("PUT", indexName); request.setJsonEntity(Strings.toString(config)); - client().performRequest(request); + client.performRequest(request); } private String fieldName(DataType type) { return type == DataType.DATETIME ? "@timestamp" : "f_" + type.esType(); } - private void typeMapping(XContentBuilder config, DataType type) throws IOException { + private void typeMapping(IndexMode indexMode, XContentBuilder config, DataType type) throws IOException { switch (type) { case COUNTER_DOUBLE, COUNTER_INTEGER, COUNTER_LONG -> config.field("type", type.esType().replace("counter_", "")) .field("time_series_metric", "counter"); @@ -246,7 +257,7 @@ private void typeMapping(XContentBuilder config, DataType type) throws IOExcepti } } - private void createAllTypesDoc(String indexName) throws IOException { + private void createAllTypesDoc(RestClient client, String indexName) throws IOException { XContentBuilder doc = JsonXContent.contentBuilder().startObject(); for (DataType type : DataType.values()) { if (supportedInIndex(type) == false) { @@ -279,7 +290,7 @@ private void createAllTypesDoc(String indexName) throws IOException { Request request = new Request("POST", indexName + "/_doc"); request.addParameter("refresh", ""); request.setJsonEntity(Strings.toString(doc)); - client().performRequest(request); + client.performRequest(request); } private Matcher expectedValue(Version version, DataType type) { @@ -321,7 +332,7 @@ private Matcher expectedValue(Version version, DataType type) { /** * Is the type supported in indices? */ - private boolean supportedInIndex(DataType t) { + private static boolean supportedInIndex(DataType t) { return switch (t) { // These are supported but implied by the index process. case OBJECT, SOURCE, DOC_DATA_TYPE, TSID_DATA_TYPE, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index a5cbd69fe603d..3d676355c213a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -200,6 +200,7 @@ public class Analyzer extends ParameterizedRuleExecutor( "Initialize", Limiter.ONCE, + new ResolveTable(), new ResolveEnrich(), new ResolveLookupTables(), From 37803628a9f4559f4f24934d8cbaa6e4c5211506 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 23 Sep 2025 16:37:58 +0000 Subject: [PATCH 09/38] [CI] Auto commit changes from spotless --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 1d0f29f594c43..6a3cd906021cb 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -88,6 +88,7 @@ protected AllSupportedFieldsTestCase(MappedFieldType.FieldExtractPreference extr protected record NodeInfo(String id, Version version) {} private static Map nodeToInfo; + private Map nodeToInfo() throws IOException { if (nodeToInfo == null) { nodeToInfo = fetchNodeToInfo(client()); From 5249481ea62a5a6883f82512b938ccae14283a7b Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 23 Sep 2025 16:38:08 +0000 Subject: [PATCH 10/38] [CI] Update transport version definitions --- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 6e7d51d3d3020..b1209b927d8a5 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -security_stats_endpoint,9168000 +inference_api_openai_embeddings_headers,9169000 From 09c4eb995179a7f68f0b996920664d33b08518a8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 23 Sep 2025 17:48:21 -0400 Subject: [PATCH 11/38] Only enable if fn or ts --- .../xpack/esql/ccq/AllSupportedFieldsIT.java | 2 +- .../qa/rest/AllSupportedFieldsTestCase.java | 62 ++++++---- .../generative/GenerativeForkRestTest.java | 5 + .../main/resources/dense_vector-bit.csv-spec | 6 +- .../main/resources/dense_vector-byte.csv-spec | 7 +- .../src/main/resources/dense_vector.csv-spec | 5 +- .../xpack/esql/analysis/Analyzer.java | 1 - .../xpack/esql/analysis/PreAnalyzer.java | 33 ++++-- .../esql/enrich/EnrichPolicyResolver.java | 40 ++++--- .../xpack/esql/session/EsqlSession.java | 4 + .../xpack/esql/session/IndexResolver.java | 43 +++++-- .../xpack/esql/analysis/AnalyzerTests.java | 112 ++++++++++++------ .../esql/type/EsqlDataTypeRegistryTests.java | 2 +- 13 files changed, 223 insertions(+), 99 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java index 0ce60186cf05a..42f23f9a3f0d7 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java @@ -51,7 +51,7 @@ public void createRemoteIndices() throws IOException { private Map remoteNodeToInfo() throws IOException { if (remoteNodeToInfo == null) { - remoteNodeToInfo = fetchNodeToInfo(remoteClient()); + remoteNodeToInfo = fetchNodeToInfo(remoteClient(), "remote_cluster"); } return remoteNodeToInfo; } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 1d0f29f594c43..5fa86d71f1dc3 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; +import org.elasticsearch.test.ListMatcher; import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; @@ -32,6 +33,7 @@ import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -40,6 +42,8 @@ import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.any; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -85,12 +89,13 @@ protected AllSupportedFieldsTestCase(MappedFieldType.FieldExtractPreference extr this.indexMode = indexMode; } - protected record NodeInfo(String id, Version version) {} + protected record NodeInfo(String cluster, String id, Version version) {} private static Map nodeToInfo; + private Map nodeToInfo() throws IOException { if (nodeToInfo == null) { - nodeToInfo = fetchNodeToInfo(client()); + nodeToInfo = fetchNodeToInfo(client(), null); } return nodeToInfo; } @@ -102,7 +107,7 @@ protected Map allNodeToInfo() throws IOException { return nodeToInfo(); } - protected static Map fetchNodeToInfo(RestClient client) throws IOException { + protected static Map fetchNodeToInfo(RestClient client, String cluster) throws IOException { Map nodeToInfo = new TreeMap<>(); Request getIds = new Request("GET", "_cat/nodes"); getIds.addParameter("h", "name,id,version"); @@ -114,7 +119,7 @@ protected static Map fetchNodeToInfo(RestClient client) throws logger.info("node: {}", line); String[] l = line.split(" "); // TODO what's the right thing to use instead of Version? - nodeToInfo.put(l[0], new NodeInfo(l[1], Version.fromString(l[2]))); + nodeToInfo.put(l[0], new NodeInfo(cluster, l[1], Version.fromString(l[2]))); } return nodeToInfo; } @@ -130,7 +135,7 @@ public final void test() throws IOException { Request request = new Request("POST", "_query"); XContentBuilder body = JsonXContent.contentBuilder().startObject(); body.field("query", """ - FROM %mode%* METADATA _id, _ignored, _index, _index_mode, _score, _source, _version + FROM *:%mode%*,%mode%* METADATA _id, _ignored, _index, _index_mode, _score, _source, _version | LIMIT 1000 """.replace("%mode%", indexMode.toString())); { @@ -156,11 +161,12 @@ public final void test() throws IOException { profileLogger.extractProfile(response, true); MapMatcher expectedColumns = matchesMap(); + Version minVersion = allNodeToInfo().values().stream().map(n -> n.version).min(Comparator.comparing(v -> v.id)).get(); for (DataType type : DataType.values()) { if (supportedInIndex(type) == false) { continue; } - expectedColumns = expectedColumns.entry(fieldName(type), expectedType(type)); + expectedColumns = expectedColumns.entry(fieldName(type), expectedType(minVersion, type)); } expectedColumns = expectedColumns.entry("_id", "keyword") .entry("_ignored", "keyword") @@ -180,9 +186,12 @@ public final void test() throws IOException { if (supportedInIndex(type) == false) { continue; } - expectedValues = expectedValues.entry(fieldName(type), expectedValue(nodeInfo.version, type)); + expectedValues = expectedValues.entry(fieldName(type), expectedValue(minVersion, nodeInfo.version, type)); } String expectedIndex = indexMode + "_" + nodeName; + if (nodeInfo.cluster != null) { + expectedIndex = nodeInfo.cluster + ":" + expectedIndex; + } expectedValues = expectedValues.entry("_id", any(String.class)) .entry("_ignored", nullValue()) .entry("_index", expectedIndex) @@ -293,7 +302,7 @@ private void createAllTypesDoc(RestClient client, String indexName) throws IOExc client.performRequest(request); } - private Matcher expectedValue(Version version, DataType type) { + private Matcher expectedValue(Version minVersion, Version version, DataType type) { return switch (type) { case BOOLEAN -> equalTo(true); case COUNTER_LONG, LONG, COUNTER_INTEGER, INTEGER, UNSIGNED_LONG, SHORT, BYTE -> equalTo(1); @@ -312,18 +321,20 @@ private Matcher expectedValue(Version version, DataType type) { case GEO_SHAPE -> equalTo("POINT (-71.34 41.12)"); case NULL -> nullValue(); case AGGREGATE_METRIC_DOUBLE -> { - // TODO this is almost certainly not the right version - if (false == version.onOrAfter(Version.V_9_0_4)) { + if (minVersion.onOrAfter(Version.V_9_2_0)) { yield nullValue(); } - // TODO why not a map? - yield equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); + Matcher expected = equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); + yield anyOf(nullValue(), expected); } case DENSE_VECTOR -> { - if (false == version.onOrAfter(Version.V_9_2_0)) { - yield matchesList().item(0.04283529).item(0.85670584).item(0.5140235); + if (minVersion.onOrAfter(Version.V_9_2_0)) { + yield nullValue(); } - yield matchesList().item(0.5).item(10.0).item(5.9999995); + Matcher> expected = version.onOrAfter(Version.V_9_2_0) + ? matchesList().item(0.5).item(10.0).item(5.9999995) + : matchesList().item(0.04283529).item(0.85670584).item(0.5140235); + yield anyOf(nullValue(), expected); } default -> throw new AssertionError("unsupported field type [" + type + "]"); }; @@ -383,18 +394,25 @@ private Map nameToValue(List names, List values) { return result; } - private String expectedType(DataType type) { + private Matcher expectedType(Version minVersion, DataType type) { return switch (type) { case COUNTER_DOUBLE, COUNTER_LONG, COUNTER_INTEGER -> { if (indexMode == IndexMode.TIME_SERIES) { - yield type.esType(); + yield equalTo(type.esType()); + } + yield equalTo(type.esType().replace("counter_", "")); + } + case BYTE, SHORT -> equalTo("integer"); + case HALF_FLOAT, SCALED_FLOAT, FLOAT -> equalTo("double"); + case NULL -> equalTo("keyword"); + // Currently unsupported without TS command or KNN function + case AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> { + if (false == minVersion.onOrAfter(Version.V_9_2_0)) { + yield either(equalTo("unsupported")).or(equalTo(type.esType())); } - yield type.esType().replace("counter_", ""); + yield equalTo("unsupported"); } - case BYTE, SHORT -> "integer"; - case HALF_FLOAT, SCALED_FLOAT, FLOAT -> "double"; - case NULL -> "keyword"; - default -> type.esType(); + default -> equalTo(type.esType()); }; } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java index 5b2ea9a525489..057e93ba3bc0e 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java @@ -54,6 +54,11 @@ protected void shouldSkipTest(String testName) throws IOException { "Tests using INSIST are not supported for now", testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName()) ); + assumeFalse( + "NOCOMMIT decide if we should keep this", + testCase.requiredCapabilities.contains(DENSE_VECTOR_FIELD_TYPE_BIT_ELEMENTS.capabilityName()) + ); + assumeTrue("Cluster needs to support FORK", hasCapabilities(adminClient(), List.of(FORK_V9.capabilityName()))); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec index 19d8892fc2367..9ddfe81b405df 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec @@ -2,6 +2,7 @@ retrieveBitVectorData required_capability: dense_vector_field_type_bit_elements FROM dense_vector +| EVAL k = SCORE(v_l2_norm(bit_vector, [1])) | KEEP id, bit_vector | SORT id ; @@ -17,7 +18,7 @@ denseBitVectorWithEval required_capability: dense_vector_field_type_bit_elements FROM dense_vector -| EVAL v = bit_vector +| EVAL v = bit_vector, k = SCORE(v_l2_norm(bit_vector, [1])) | KEEP id, v | SORT id ; @@ -34,8 +35,9 @@ required_capability: dense_vector_field_type_bit_elements FROM dense_vector | EVAL v = bit_vector +| EVAL k = SCORE(v_l2_norm(bit_vector, [1])) | RENAME v AS new_vector -| DROP float_vector, byte_vector, bit_vector +| DROP float_vector, byte_vector, bit_vector, k | SORT id ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec index b9caf3c59bc81..3767b899b5214 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec @@ -2,6 +2,7 @@ retrieveByteVectorData required_capability: dense_vector_field_type_byte_elements FROM dense_vector +| EVAL k = SCORE(v_l2_norm(byte_vector, [1])) | KEEP id, byte_vector | SORT id ; @@ -18,6 +19,7 @@ required_capability: dense_vector_field_type_byte_elements FROM dense_vector | EVAL v = byte_vector +| EVAL k = SCORE(v_l2_norm(byte_vector, [1])) | KEEP id, v | SORT id ; @@ -33,9 +35,10 @@ denseByteVectorWithRenameAndDrop required_capability: dense_vector_field_type_byte_elements FROM dense_vector -| EVAL v = byte_vector +| EVAL v = byte_vector +| EVAL k = SCORE(v_l2_norm(byte_vector, [1])) | RENAME v AS new_vector -| DROP float_vector, byte_vector, bit_vector +| DROP float_vector, byte_vector, bit_vector, k | SORT id ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec index c8a24d84ce72a..4f0d7fd60f248 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec @@ -2,6 +2,7 @@ retrieveDenseVectorData required_capability: dense_vector_field_type FROM dense_vector +| EVAL k = SCORE(v_l2_norm(float_vector, [1])) | KEEP id, float_vector | SORT id ; @@ -18,6 +19,7 @@ required_capability: dense_vector_field_type FROM dense_vector | EVAL v = float_vector +| EVAL k = SCORE(v_l2_norm(float_vector, [1])) | KEEP id, v | SORT id ; @@ -34,8 +36,9 @@ required_capability: dense_vector_field_type FROM dense_vector | EVAL v = float_vector +| EVAL k = SCORE(v_l2_norm(float_vector, [1])) | RENAME v AS new_vector -| DROP float_vector, byte_vector, bit_vector +| DROP float_vector, byte_vector, bit_vector, k | SORT id ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 3d676355c213a..a5cbd69fe603d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -200,7 +200,6 @@ public class Analyzer extends ParameterizedRuleExecutor( "Initialize", Limiter.ONCE, - new ResolveTable(), new ResolveEnrich(), new ResolveLookupTables(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java index 07af4e0d4d4ca..099e70dbba5da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java @@ -8,11 +8,14 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.xpack.esql.core.util.Holder; +import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.esql.session.IndexResolver; import java.util.ArrayList; import java.util.List; @@ -22,8 +25,14 @@ */ public class PreAnalyzer { - public record PreAnalysis(IndexMode indexMode, IndexPattern indexPattern, List enriches, List lookupIndices) { - public static final PreAnalysis EMPTY = new PreAnalysis(null, null, List.of(), List.of()); + public record PreAnalysis( + IndexMode indexMode, + IndexPattern indexPattern, + List enriches, + List lookupIndices, + boolean supportsDenseVector + ) { + public static final PreAnalysis EMPTY = new PreAnalysis(null, null, List.of(), List.of(), false); } public PreAnalysis preAnalyze(LogicalPlan plan) { @@ -35,13 +44,9 @@ public PreAnalysis preAnalyze(LogicalPlan plan) { } protected PreAnalysis doPreAnalyze(LogicalPlan plan) { - Holder indexMode = new Holder<>(); Holder index = new Holder<>(); - - List unresolvedEnriches = new ArrayList<>(); List lookupIndices = new ArrayList<>(); - plan.forEachUp(UnresolvedRelation.class, p -> { if (p.indexMode() == IndexMode.LOOKUP) { lookupIndices.add(p.indexPattern()); @@ -53,11 +58,25 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) { } }); + List unresolvedEnriches = new ArrayList<>(); plan.forEachUp(Enrich.class, unresolvedEnriches::add); + Holder supportsDenseVector = new Holder<>(false); + plan.forEachDown(p -> p.forEachExpression(UnresolvedFunction.class, fn -> { + if (fn.name().equalsIgnoreCase("knn") + || fn.name().equalsIgnoreCase("v_cosine") + || fn.name().equalsIgnoreCase("v_hamming") + || fn.name().equalsIgnoreCase("v_l1_norm") + || fn.name().equalsIgnoreCase("v_l2_norm") + || fn.name().equalsIgnoreCase("v_dot_product") + || fn.name().equalsIgnoreCase("v_magnitude")) { + supportsDenseVector.set(true); + } + })); + // mark plan as preAnalyzed (if it were marked, there would be no analysis) plan.forEachUp(LogicalPlan::setPreAnalyzed); - return new PreAnalysis(indexMode.get(), index.get(), unresolvedEnriches, lookupIndices); + return new PreAnalysis(indexMode.get(), index.get(), unresolvedEnriches, lookupIndices, supportsDenseVector.get()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java index d9314f1a0611d..a1537dc63bfd4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java @@ -444,22 +444,30 @@ public void messageReceived(LookupRequest request, TransportChannel channel, Tas } try (ThreadContext.StoredContext ignored = threadContext.stashWithOrigin(ClientHelper.ENRICH_ORIGIN)) { String indexName = EnrichPolicy.getBaseName(policyName); - indexResolver.resolveAsMergedMapping(indexName, IndexResolver.ALL_FIELDS, null, false, refs.acquire(indexResult -> { - if (indexResult.isValid() && indexResult.get().concreteIndices().size() == 1) { - EsIndex esIndex = indexResult.get(); - var concreteIndices = Map.of(request.clusterAlias, Iterables.get(esIndex.concreteIndices(), 0)); - var resolved = new ResolvedEnrichPolicy( - p.getMatchField(), - p.getType(), - p.getEnrichFields(), - concreteIndices, - esIndex.mapping() - ); - resolvedPolices.put(policyName, resolved); - } else { - failures.put(policyName, indexResult.toString()); - } - })); + indexResolver.resolveAsMergedMapping( + indexName, + IndexResolver.ALL_FIELDS, + null, + false, + false, + false, + refs.acquire(indexResult -> { + if (indexResult.isValid() && indexResult.get().concreteIndices().size() == 1) { + EsIndex esIndex = indexResult.get(); + var concreteIndices = Map.of(request.clusterAlias, Iterables.get(esIndex.concreteIndices(), 0)); + var resolved = new ResolvedEnrichPolicy( + p.getMatchField(), + p.getType(), + p.getEnrichFields(), + concreteIndices, + esIndex.mapping() + ); + resolvedPolices.put(policyName, resolved); + } else { + failures.put(policyName, indexResult.toString()); + } + }) + ); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index b62da0bce2564..541aa057b668b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -440,6 +440,8 @@ private void preAnalyzeLookupIndex( result.wildcardJoinIndices().contains(localPattern) ? IndexResolver.ALL_FIELDS : result.fieldNames, null, false, + false, + false, listener.map(indexResolution -> receiveLookupIndexResolution(result, localPattern, executionInfo, indexResolution)) ); } @@ -657,6 +659,8 @@ private void preAnalyzeMainIndices( default -> requestFilter; }, preAnalysis.indexMode() == IndexMode.TIME_SERIES, + preAnalysis.indexMode() == IndexMode.TIME_SERIES, + preAnalysis.supportsDenseVector(), listener.delegateFailure((l, indexResolution) -> { l.onResponse(result.withIndexResolution(indexResolution)); }) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 2771afb6ef3ca..03d9226e69d1f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; @@ -42,7 +43,9 @@ import java.util.TreeMap; import java.util.TreeSet; +import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; @@ -82,25 +85,34 @@ public void resolveAsMergedMapping( Set fieldNames, QueryBuilder requestFilter, boolean includeAllDimensions, + boolean supportsAggregateMetricDouble, + boolean supportsDenseVector, ActionListener listener ) { client.execute( EsqlResolveFieldsAction.TYPE, createFieldCapsRequest(indexWildcard, fieldNames, requestFilter, includeAllDimensions), - listener.delegateFailureAndWrap((l, response) -> l.onResponse(mergedMappings(indexWildcard, response))) + listener.delegateFailureAndWrap( + (l, response) -> l.onResponse( + mergedMappings(indexWildcard, new FieldsInfo(response, supportsAggregateMetricDouble, supportsDenseVector)) + ) + ) ); } + public record FieldsInfo(FieldCapabilitiesResponse caps, boolean supportAggregateMetricDouble, boolean supportDenseVector) {} + // public for testing only - public static IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResponse fieldCapsResponse) { + public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fieldsInfo) { + LogManager.getLogger(IndexResolver.class).error("NOCOMMIT {} {}", fieldsInfo.supportDenseVector, fieldsInfo.supportAggregateMetricDouble); assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker - var numberOfIndices = fieldCapsResponse.getIndexResponses().size(); - if (fieldCapsResponse.getIndexResponses().isEmpty()) { + int numberOfIndices = fieldsInfo.caps.getIndexResponses().size(); + if (numberOfIndices == 0) { return IndexResolution.notFound(indexPattern); } // For each field name, store a list of the field caps responses from each index - var collectedFieldCaps = collectFieldCaps(fieldCapsResponse); + var collectedFieldCaps = collectFieldCaps(fieldsInfo.caps); Map fieldsCaps = collectedFieldCaps.fieldsCaps; Map indexMappingHashDuplicates = collectedFieldCaps.indexMappingHashDuplicates; @@ -138,7 +150,7 @@ public static IndexResolution mergedMappings(String indexPattern, FieldCapabilit var fieldCap = fieldsCaps.get(fullName); List fcs = fieldCap.fieldCapabilities; EsField field = firstUnsupportedParent == null - ? createField(fieldCapsResponse, name, fullName, fcs, isAlias) + ? createField(fieldsInfo, name, fullName, fcs, isAlias) : new UnsupportedEsField( fullName, firstUnsupportedParent.getOriginalTypes(), @@ -152,13 +164,13 @@ public static IndexResolution mergedMappings(String indexPattern, FieldCapabilit } } - Map concreteIndices = Maps.newMapWithExpectedSize(fieldCapsResponse.getIndexResponses().size()); - for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) { + Map concreteIndices = Maps.newMapWithExpectedSize(fieldsInfo.caps.getIndexResponses().size()); + for (FieldCapabilitiesIndexResponse ir : fieldsInfo.caps.getIndexResponses()) { concreteIndices.put(ir.getIndexName(), ir.getIndexMode()); } boolean allEmpty = true; - for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) { + for (FieldCapabilitiesIndexResponse ir : fieldsInfo.caps.getIndexResponses()) { allEmpty &= ir.get().isEmpty(); } // If all the mappings are empty we return an empty set of resolved indices to line up with QL @@ -168,7 +180,7 @@ public static IndexResolution mergedMappings(String indexPattern, FieldCapabilit // for fields that do not exist in the index (but the index has a mapping) will result in "VerificationException Unknown column" // errors. var index = new EsIndex(indexPattern, rootFields, allEmpty ? Map.of() : concreteIndices, partiallyUnmappedFields); - var failures = EsqlCCSUtils.groupFailuresPerCluster(fieldCapsResponse.getFailures()); + var failures = EsqlCCSUtils.groupFailuresPerCluster(fieldsInfo.caps.getFailures()); return IndexResolution.valid(index, concreteIndices.keySet(), failures); } @@ -215,7 +227,7 @@ private static CollectedFieldCaps collectFieldCaps(FieldCapabilitiesResponse fie } private static EsField createField( - FieldCapabilitiesResponse fieldCapsResponse, + FieldsInfo fieldsInfo, String name, String fullName, List fcs, @@ -224,12 +236,17 @@ private static EsField createField( IndexFieldCapabilities first = fcs.get(0); List rest = fcs.subList(1, fcs.size()); DataType type = EsqlDataTypeRegistry.INSTANCE.fromEs(first.type(), first.metricType()); + type = switch (type) { + case AGGREGATE_METRIC_DOUBLE -> fieldsInfo.supportAggregateMetricDouble ? AGGREGATE_METRIC_DOUBLE : UNSUPPORTED; + case DENSE_VECTOR -> fieldsInfo.supportDenseVector ? DENSE_VECTOR : UNSUPPORTED; + default -> type; + }; boolean aggregatable = first.isAggregatable(); EsField.TimeSeriesFieldType timeSeriesFieldType = EsField.TimeSeriesFieldType.fromIndexFieldCapabilities(first); if (rest.isEmpty() == false) { for (IndexFieldCapabilities fc : rest) { if (first.metricType() != fc.metricType()) { - return conflictingMetricTypes(name, fullName, fieldCapsResponse); + return conflictingMetricTypes(name, fullName, fieldsInfo.caps); } try { timeSeriesFieldType = timeSeriesFieldType.merge(EsField.TimeSeriesFieldType.fromIndexFieldCapabilities(fc)); @@ -239,7 +256,7 @@ private static EsField createField( } for (IndexFieldCapabilities fc : rest) { if (type != EsqlDataTypeRegistry.INSTANCE.fromEs(fc.type(), fc.metricType())) { - return conflictingTypes(name, fullName, fieldCapsResponse); + return conflictingTypes(name, fullName, fieldsInfo.caps); } } for (IndexFieldCapabilities fc : rest) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 911bd45edd800..e9601232808e8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3172,12 +3172,16 @@ public void testResolveInsist_multiIndexFieldPartiallyMappedWithSingleKeywordTyp IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("keyword")), - fieldCapabilitiesIndexResponse("bar", Map.of()) + new IndexResolver.FieldsInfo( + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("keyword")), + fieldCapabilitiesIndexResponse("bar", Map.of()) + ), + List.of() ), - List.of() + true, + true ) ); @@ -3195,9 +3199,16 @@ public void testResolveInsist_multiIndexFieldExistsWithSingleTypeButIsNotKeyword IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", - new FieldCapabilitiesResponse( - List.of(fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), fieldCapabilitiesIndexResponse("bar", Map.of())), - List.of() + new IndexResolver.FieldsInfo( + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", Map.of()) + ), + List.of() + ), + true, + true ) ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); @@ -3216,13 +3227,17 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyw IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), - fieldCapabilitiesIndexResponse("bazz", Map.of()) + new IndexResolver.FieldsInfo( + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), + fieldCapabilitiesIndexResponse("bazz", Map.of()) + ), + List.of() ), - List.of() + true, + true ) ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); @@ -3240,12 +3255,16 @@ public void testResolveInsist_multiIndexSameMapping_fieldIsMapped() { IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("long")) + new IndexResolver.FieldsInfo( + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("long")) + ), + List.of() ), - List.of() + true, + true ) ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); @@ -3261,14 +3280,18 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithKe IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), - fieldCapabilitiesIndexResponse("bazz", messageResponseMap("keyword")), - fieldCapabilitiesIndexResponse("qux", Map.of()) + new IndexResolver.FieldsInfo( + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), + fieldCapabilitiesIndexResponse("bazz", messageResponseMap("keyword")), + fieldCapabilitiesIndexResponse("qux", Map.of()) + ), + List.of() ), - List.of() + true, + true ) ); var plan = analyze("FROM foo, bar | INSIST_🐔 message", analyzer(resolution, TEST_VERIFIER)); @@ -3286,13 +3309,17 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), - fieldCapabilitiesIndexResponse("bazz", Map.of()) + new IndexResolver.FieldsInfo( + new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), + fieldCapabilitiesIndexResponse("bazz", Map.of()) + ), + List.of() ), - List.of() + true, + true ) ); VerificationException e = expectThrows( @@ -3306,6 +3333,25 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa ); } + public void testResolveDenseVector() { + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of(fieldCapabilitiesIndexResponse("foo", Map.of("v", new IndexFieldCapabilitiesBuilder("v", "dense_vector").build()))), + List.of() + ); + { + IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); + var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); + assertThat(plan.output(), hasSize(1)); + assertThat(plan.output().getFirst().dataType(), equalTo(DENSE_VECTOR)); + } + { + IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, false)); + var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); + assertThat(plan.output(), hasSize(1)); + assertThat(plan.output().getFirst().dataType(), equalTo(UNSUPPORTED)); + } + } + public void testBasicFork() { LogicalPlan plan = analyze(""" from test @@ -3753,7 +3799,7 @@ private static LogicalPlan analyzeWithEmptyFieldCapsResponse(String query) throw List idxResponses = List.of( new FieldCapabilitiesIndexResponse("idx", "idx", Map.of(), true, IndexMode.STANDARD) ); - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); + IndexResolver.FieldsInfo caps = new IndexResolver.FieldsInfo(new FieldCapabilitiesResponse(idxResponses, List.of()), true, true); IndexResolution resolution = IndexResolver.mergedMappings("test*", caps); var analyzer = analyzer(resolution, TEST_VERIFIER, configuration(query)); return analyze(query, analyzer); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index e8ccda09f85df..fbb22c49af331 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -54,7 +54,7 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); // IndexResolver uses EsqlDataTypeRegistry directly - IndexResolution resolution = IndexResolver.mergedMappings("idx-*", caps); + IndexResolution resolution = IndexResolver.mergedMappings("idx-*", new IndexResolver.FieldsInfo(caps, true, true)); EsField f = resolution.get().mapping().get(field); assertThat(f.getDataType(), equalTo(expected)); } From d25151ca787ad7a83fc31cebce88c1e318e3241e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 23 Sep 2025 22:01:55 +0000 Subject: [PATCH 12/38] [CI] Auto commit changes from spotless --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 1 - .../xpack/esql/qa/rest/generative/GenerativeForkRestTest.java | 1 - .../org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java | 2 -- .../org/elasticsearch/xpack/esql/session/IndexResolver.java | 3 ++- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 5fa86d71f1dc3..372893bbd6afe 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; -import org.elasticsearch.test.ListMatcher; import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java index 057e93ba3bc0e..e23ecebc99c56 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java @@ -59,7 +59,6 @@ protected void shouldSkipTest(String testName) throws IOException { testCase.requiredCapabilities.contains(DENSE_VECTOR_FIELD_TYPE_BIT_ELEMENTS.capabilityName()) ); - assumeTrue("Cluster needs to support FORK", hasCapabilities(adminClient(), List.of(FORK_V9.capabilityName()))); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java index 099e70dbba5da..d15c3466f7f4d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java @@ -8,14 +8,12 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.logging.LogManager; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; -import org.elasticsearch.xpack.esql.session.IndexResolver; import java.util.ArrayList; import java.util.List; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 03d9226e69d1f..a53a309372a58 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -104,7 +104,8 @@ public record FieldsInfo(FieldCapabilitiesResponse caps, boolean supportAggregat // public for testing only public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fieldsInfo) { - LogManager.getLogger(IndexResolver.class).error("NOCOMMIT {} {}", fieldsInfo.supportDenseVector, fieldsInfo.supportAggregateMetricDouble); + LogManager.getLogger(IndexResolver.class) + .error("NOCOMMIT {} {}", fieldsInfo.supportDenseVector, fieldsInfo.supportAggregateMetricDouble); assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker int numberOfIndices = fieldsInfo.caps.getIndexResponses().size(); if (numberOfIndices == 0) { From 280da8a8bf16cb680590b04804a23b4470e11e86 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 23 Sep 2025 19:29:15 -0400 Subject: [PATCH 13/38] impl --- .../xpack/esql/analysis/PreAnalyzer.java | 18 +++- .../xpack/esql/session/EsqlSession.java | 2 +- .../rest-api-spec/test/esql/40_tsdb.yml | 81 ++++++++++++++---- .../test/esql/40_unsupported_types.yml | 26 +++--- .../rest-api-spec/test/esql/46_downsample.yml | 84 ++++++++++++------- 5 files changed, 146 insertions(+), 65 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java index 099e70dbba5da..dc0e1f9836c1a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java @@ -8,14 +8,12 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.logging.LogManager; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; -import org.elasticsearch.xpack.esql.session.IndexResolver; import java.util.ArrayList; import java.util.List; @@ -30,9 +28,10 @@ public record PreAnalysis( IndexPattern indexPattern, List enriches, List lookupIndices, + boolean supportsAggregateMetricDouble, boolean supportsDenseVector ) { - public static final PreAnalysis EMPTY = new PreAnalysis(null, null, List.of(), List.of(), false); + public static final PreAnalysis EMPTY = new PreAnalysis(null, null, List.of(), List.of(), false, false); } public PreAnalysis preAnalyze(LogicalPlan plan) { @@ -61,6 +60,7 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) { List unresolvedEnriches = new ArrayList<>(); plan.forEachUp(Enrich.class, unresolvedEnriches::add); + Holder supportsAggregateMetricDouble = new Holder<>(false); Holder supportsDenseVector = new Holder<>(false); plan.forEachDown(p -> p.forEachExpression(UnresolvedFunction.class, fn -> { if (fn.name().equalsIgnoreCase("knn") @@ -72,11 +72,21 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) { || fn.name().equalsIgnoreCase("v_magnitude")) { supportsDenseVector.set(true); } + if (fn.name().equalsIgnoreCase("to_aggregate_metric_double")) { + supportsAggregateMetricDouble.set(true); + } })); // mark plan as preAnalyzed (if it were marked, there would be no analysis) plan.forEachUp(LogicalPlan::setPreAnalyzed); - return new PreAnalysis(indexMode.get(), index.get(), unresolvedEnriches, lookupIndices, supportsDenseVector.get()); + return new PreAnalysis( + indexMode.get(), + index.get(), + unresolvedEnriches, + lookupIndices, + indexMode.get() == IndexMode.TIME_SERIES || supportsAggregateMetricDouble.get(), + supportsDenseVector.get() + ); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 541aa057b668b..063e1b5785555 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -659,7 +659,7 @@ private void preAnalyzeMainIndices( default -> requestFilter; }, preAnalysis.indexMode() == IndexMode.TIME_SERIES, - preAnalysis.indexMode() == IndexMode.TIME_SERIES, + preAnalysis.supportsAggregateMetricDouble(), preAnalysis.supportsDenseVector(), listener.delegateFailure((l, indexResolution) -> { l.onResponse(result.withIndexResolution(indexResolution)); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index e4a6ebc966a8a..e93bd89bf7e1a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -231,7 +231,10 @@ filter on counter without cast: catch: bad_request esql.query: body: - query: 'from test | where k8s.pod.network.tx == 1434577921' + query: | + FROM test + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | WHERE k8s.pod.network.tx == 1434577921 --- cast counter then filter: @@ -263,7 +266,12 @@ sort on counter without cast: catch: /cannot sort on counter_long/ esql.query: body: - query: 'from test | KEEP k8s.pod.network.tx | sort k8s.pod.network.tx | limit 1' + query: | + FROM test + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | KEEP k8s.pod.network.tx + | SORT k8s.pod.network.tx + | LIMIT 1 --- cast then sort on counter: @@ -289,7 +297,10 @@ from doc with aggregate_metric_double: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'from test2' + query: | + FROM test2 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | DROP a - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} @@ -318,7 +329,10 @@ stats on aggregate_metric_double: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test2 | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric)' + query: | + FROM test2 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) - length: {values: 1} - length: {values.0: 4} - match: {columns.0.name: "max(agg_metric)"} @@ -349,9 +363,11 @@ grouping stats on aggregate_metric_double: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: "FROM test2 - | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY dim - | SORT dim" + query: | + FROM test2 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY dim + | SORT dim - length: {values: 2} - length: {values.0: 5} - match: {columns.0.name: "max(agg_metric)"} @@ -390,7 +406,11 @@ sorting with aggregate_metric_double with partial submetrics: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test3 | SORT @timestamp | KEEP @timestamp, agg_metric' + query: | + FROM test3 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | SORT @timestamp + | KEEP @timestamp, agg_metric - length: {values: 4} - length: {values.0: 2} @@ -421,7 +441,11 @@ aggregate_metric_double unsortable: catch: /cannot sort on aggregate_metric_double/ esql.query: body: - query: 'FROM test2 | sort agg_metric' + query: | + FROM test2 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | DROP a + | SORT agg_metric --- stats on aggregate_metric_double with partial submetrics: @@ -438,7 +462,11 @@ stats on aggregate_metric_double with partial submetrics: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test3 | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY k8s.pod.uid | SORT k8s.pod.uid' + query: | + FROM test3 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY k8s.pod.uid + | SORT k8s.pod.uid - length: {values: 2} - length: {values.0: 5} @@ -478,7 +506,10 @@ stats on aggregate_metric_double missing min and max: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test4 | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric)' + query: | + FROM test4 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) - length: {values: 1} - length: {values.0: 4} @@ -510,7 +541,10 @@ render aggregate_metric_double when missing min and max: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test4 | KEEP agg_metric' + query: | + FROM test4 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | KEEP agg_metric - length: {values: 1} - length: {values.0: 1} @@ -534,7 +568,11 @@ render aggregate_metric_double when missing value: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test3 | WHERE @timestamp == "2021-04-28T19:51:04.467Z" | KEEP agg_metric' + query: | + FROM test3 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | WHERE @timestamp == "2021-04-28T19:51:04.467Z" + | KEEP agg_metric - length: {values: 1} - length: {values.0: 1} @@ -558,7 +596,11 @@ to_string aggregate_metric_double: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test4 | EVAL agg = to_string(agg_metric) | KEEP agg' + query: | + FROM test4 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL agg = to_string(agg_metric) + | KEEP agg - length: {values: 1} - length: {values.0: 1} @@ -581,7 +623,10 @@ from index pattern unsupported counter: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test*' + query: | + FROM test* + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | DROP a - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} @@ -724,7 +769,11 @@ avg of aggregate_metric_double: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test2 | STATS avg = avg(agg_metric) | KEEP avg' + query: | + FROM test2 + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | STATS avg = avg(agg_metric) + | KEEP avg - length: {values: 1} - length: {values.0: 1} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index 9a49496866095..cd6c4a1d0daaf 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -157,8 +157,7 @@ unsupported: query: 'from test' - match: { columns.0.name: aggregate_metric_double } - - match: { columns.0.type: aggregate_metric_double } - - is_false: columns.0.original_types + - match: { columns.0.type: unsupported } - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } - match: { columns.1.original_types: [binary] } @@ -170,7 +169,7 @@ unsupported: - match: { columns.4.name: date_range } - match: { columns.4.type: unsupported } - match: { columns.5.name: dense_vector } - - match: { columns.5.type: dense_vector } + - match: { columns.5.type: unsupported } - match: { columns.6.name: double_range } - match: { columns.6.type: unsupported } - match: { columns.7.name: float_range } @@ -219,14 +218,12 @@ unsupported: - match: { columns.28.type: integer } - length: { values: 1 } - - match: { values.0.0: '{"min":1.0,"max":3.0,"sum":10.1,"value_count":5}' } + - match: { values.0.0: null } - match: { values.0.1: null } - match: { values.0.2: null } - match: { values.0.3: "2015-01-01T12:10:30.123456789Z" } - match: { values.0.4: null } - - match: { values.0.5.0: 0.5 } - - match: { values.0.5.1: 10.0 } - - match: { values.0.5.2: 6.0 } + - match: { values.0.5: null } - match: { values.0.6: null } - match: { values.0.7: null } - match: { values.0.8: "POINT (10.0 12.0)" } @@ -258,8 +255,7 @@ unsupported: body: query: 'from test | limit 0' - match: { columns.0.name: aggregate_metric_double } - - match: { columns.0.type: aggregate_metric_double } - - is_false: columns.0.original_types + - match: { columns.0.type: unsupported } - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } - match: { columns.1.original_types: [binary] } @@ -271,7 +267,7 @@ unsupported: - match: { columns.4.name: date_range } - match: { columns.4.type: unsupported } - match: { columns.5.name: dense_vector } - - match: { columns.5.type: dense_vector } + - match: { columns.5.type: unsupported } - match: { columns.6.name: double_range } - match: { columns.6.type: unsupported } - match: { columns.7.name: float_range } @@ -354,7 +350,7 @@ unsupported with sort: query: 'from test | sort some_doc.bar' - match: { columns.0.name: aggregate_metric_double } - - match: { columns.0.type: aggregate_metric_double } + - match: { columns.0.type: unsupported } - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } - match: { columns.2.name: completion } @@ -364,7 +360,7 @@ unsupported with sort: - match: { columns.4.name: date_range } - match: { columns.4.type: unsupported } - match: { columns.5.name: dense_vector } - - match: { columns.5.type: dense_vector } + - match: { columns.5.type: unsupported } - match: { columns.6.name: double_range } - match: { columns.6.type: unsupported } - match: { columns.7.name: float_range } @@ -413,14 +409,12 @@ unsupported with sort: - match: { columns.28.type: integer } - length: { values: 1 } - - match: { values.0.0: '{"min":1.0,"max":3.0,"sum":10.1,"value_count":5}' } + - match: { values.0.0: null } - match: { values.0.1: null } - match: { values.0.2: null } - match: { values.0.3: "2015-01-01T12:10:30.123456789Z" } - match: { values.0.4: null } - - match: { values.0.5.0: 0.5 } - - match: { values.0.5.1: 10.0 } - - match: { values.0.5.2: 6.0 } + - match: { values.0.5: null } - match: { values.0.6: null } - match: { values.0.7: null } - match: { values.0.8: "POINT (10.0 12.0)" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml index 6320d81e7f4ac..95ff143de8de8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml @@ -98,9 +98,12 @@ setup: - do: esql.query: body: - query: "FROM test-downsample | - STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx) - | LIMIT 100" + query: | + FROM test-downsample + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | DROP a + | STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx) + | LIMIT 100 - length: {values: 1} - length: {values.0: 4} @@ -140,7 +143,13 @@ setup: - do: esql.query: body: - query: "FROM test-downsample | WHERE @timestamp == \"2021-04-28T19:00:00.000Z\" | KEEP k8s.pod.network.rx | LIMIT 100" + query: | + FROM test-downsample + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | DROP a + | WHERE @timestamp == "2021-04-28T19:00:00.000Z" + | KEEP k8s.pod.network.rx + | LIMIT 100 - length: {values: 1} - length: {values.0: 1} - match: {columns.0.name: "k8s.pod.network.rx"} @@ -231,11 +240,12 @@ setup: - do: esql.query: body: - query: "FROM test-* | - WHERE k8s.pod.uid == \"947e4ced-1786-4e53-9e0c-5c447e959507\" | - EVAL rx = to_aggregate_metric_double(k8s.pod.network.rx) | - STATS max(rx), min(rx), sum(rx), count(rx) | - LIMIT 100" + query: | + FROM test-* + | WHERE k8s.pod.uid == "947e4ced-1786-4e53-9e0c-5c447e959507" + | EVAL rx = to_aggregate_metric_double(k8s.pod.network.rx) + | STATS max(rx), min(rx), sum(rx), count(rx) + | LIMIT 100 - length: {values: 1} - length: {values.0: 4} @@ -336,10 +346,12 @@ setup: - do: esql.query: body: - query: "FROM test-* | - WHERE k8s.pod.uid == \"947e4ced-1786-4e53-9e0c-5c447e959507\" | - STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx), avg(k8s.pod.network.rx) | - LIMIT 100" + query: | + FROM test-* + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | WHERE k8s.pod.uid == \"947e4ced-1786-4e53-9e0c-5c447e959507\" + | STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx), avg(k8s.pod.network.rx) + | LIMIT 100 - length: {values: 1} - length: {values.0: 5} @@ -457,12 +469,14 @@ setup: - do: esql.query: body: - query: "TS test-* | - STATS avg = sum(avg_over_time(k8s.pod.network.rx)), - count = sum(count_over_time(k8s.pod.network.rx)), - sum = sum(sum_over_time(k8s.pod.network.rx)) - BY time_bucket = bucket(@timestamp, 1 hour) | - SORT time_bucket | LIMIT 10" + query: | + TS test-* + | STATS avg = sum(avg_over_time(k8s.pod.network.rx)), + count = sum(count_over_time(k8s.pod.network.rx)), + sum = sum(sum_over_time(k8s.pod.network.rx)) + BY time_bucket = bucket(@timestamp, 1 hour) + | SORT time_bucket + | LIMIT 10 - length: {values: 4} - length: {values.0: 4} @@ -579,13 +593,14 @@ setup: - do: esql.query: body: - query: "TS test-* | - STATS avg = sum(avg_over_time(k8s.pod.network.rx)), - count = sum(count_over_time(k8s.pod.network.rx)), - sum = sum(sum_over_time(k8s.pod.network.rx)) - BY k8s.pod.name, time_bucket = bucket(@timestamp, 1 hour) | - SORT time_bucket, k8s.pod.name | - LIMIT 10" + query: | + TS test-* + | STATS avg = sum(avg_over_time(k8s.pod.network.rx)), + count = sum(count_over_time(k8s.pod.network.rx)), + sum = sum(sum_over_time(k8s.pod.network.rx)) + BY k8s.pod.name, time_bucket = bucket(@timestamp, 1 hour) + | SORT time_bucket, k8s.pod.name + |LIMIT 10 - length: {values: 6} - length: {values.0: 5} @@ -671,7 +686,13 @@ setup: - do: esql.query: body: - query: "FROM test-* | SORT some_field, @timestamp, k8s.pod.uid | KEEP k8s.pod.network.rx, some_field, @timestamp | LIMIT 10" + query: | + FROM test-* + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | DROP a + | SORT some_field, @timestamp, k8s.pod.uid + | KEEP k8s.pod.network.rx, some_field, @timestamp + | LIMIT 10 - length: {values: 5} - length: {values.0: 3} @@ -721,7 +742,14 @@ setup: - do: esql.query: body: - query: "FROM test-downsample | MV_EXPAND k8s.pod.network.rx | SORT @timestamp, k8s.pod.uid | KEEP k8s.pod.network.rx, @timestamp | LIMIT 10" + query: | + FROM test-downsample + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | DROP a + | MV_EXPAND k8s.pod.network.rx + | SORT @timestamp, k8s.pod.uid + | KEEP k8s.pod.network.rx, @timestamp + | LIMIT 10 - length: {values: 4} - length: {values.0: 2} From b9fee12e69a6817421b66f7f5b318f48d18ae949 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 23 Sep 2025 19:36:44 -0400 Subject: [PATCH 14/38] Hack --- .../org/elasticsearch/xpack/esql/session/IndexResolver.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index a53a309372a58..ff042c5e7d870 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.logging.LogManager; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; @@ -104,8 +103,6 @@ public record FieldsInfo(FieldCapabilitiesResponse caps, boolean supportAggregat // public for testing only public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fieldsInfo) { - LogManager.getLogger(IndexResolver.class) - .error("NOCOMMIT {} {}", fieldsInfo.supportDenseVector, fieldsInfo.supportAggregateMetricDouble); assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker int numberOfIndices = fieldsInfo.caps.getIndexResponses().size(); if (numberOfIndices == 0) { From a1d2389c8cce63f22cd74bfa6b3fc211dea542de Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 09:06:28 -0400 Subject: [PATCH 15/38] Change error --- .../org/elasticsearch/xpack/esql/core/type/DataType.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index e837d928dac78..c4e42524ef661 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -755,7 +755,14 @@ public DataType counter() { @Override public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(createdVersion) == false) { - throw new IllegalStateException( + /* + * TODO when we implement version aware planning flip this to an IllegalStateException + * so we throw a 500 error. It'll be our bug then. Right now it's a sign that the user + * tried to do something like `KNN(dense_vector_field, [1, 2])` against an old node. + * Like, during the rolling upgrade that enables KNN or to a remote cluster that has + * not yet been upgraded. + */ + throw new IllegalArgumentException( "remote node at version [" + out.getTransportVersion() + "] doesn't understand data type [" + this + "]" ); } From ef0c2691f2f03a123e37b8aa7b9dbf7b28d82ace Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 10:05:45 -0400 Subject: [PATCH 16/38] Fetch dense_Vector --- .../qa/rest/AllSupportedFieldsTestCase.java | 108 ++++++++++++------ 1 file changed, 73 insertions(+), 35 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 372893bbd6afe..18b1ba417ffba 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -130,34 +130,13 @@ public void createIndices() throws IOException { } } - public final void test() throws IOException { - Request request = new Request("POST", "_query"); - XContentBuilder body = JsonXContent.contentBuilder().startObject(); - body.field("query", """ - FROM *:%mode%*,%mode%* METADATA _id, _ignored, _index, _index_mode, _score, _source, _version + public final void testFetchAll() throws IOException { + Map response = esql(""" + , _id, _ignored, _index_mode, _score, _source, _version | LIMIT 1000 - """.replace("%mode%", indexMode.toString())); - { - body.startObject("pragma"); - if (extractPreference != null) { - body.field("field_extract_preference", extractPreference); - } - body.endObject(); - } - body.field("accept_pragma_risks", "true"); - body.field("profile", true); - body.field("include_ccs_metadata", true); - body.endObject(); - request.setJsonEntity(Strings.toString(body)); - - Map response = responseAsMap(client().performRequest(request)); - if ((Boolean) response.get("is_partial")) { - throw new AssertionError("partial results: " + response); - } - + """); List columns = (List) response.get("columns"); List values = (List) response.get("values"); - profileLogger.extractProfile(response, true); MapMatcher expectedColumns = matchesMap(); Version minVersion = allNodeToInfo().values().stream().map(n -> n.version).min(Comparator.comparing(v -> v.id)).get(); @@ -187,23 +166,68 @@ public final void test() throws IOException { } expectedValues = expectedValues.entry(fieldName(type), expectedValue(minVersion, nodeInfo.version, type)); } - String expectedIndex = indexMode + "_" + nodeName; - if (nodeInfo.cluster != null) { - expectedIndex = nodeInfo.cluster + ":" + expectedIndex; - } expectedValues = expectedValues.entry("_id", any(String.class)) .entry("_ignored", nullValue()) - .entry("_index", expectedIndex) + .entry("_index", expectedIndex(nodeName, nodeInfo)) .entry("_index_mode", indexMode.toString()) .entry("_score", 0.0) .entry("_source", matchesMap().extraOk()) .entry("_version", 1); - expectedAllValues = expectedAllValues.entry(expectedIndex, expectedValues); + expectedAllValues = expectedAllValues.entry(expectedIndex(nodeName, nodeInfo), expectedValues); } assertMap(indexToRow(columns, values), expectedAllValues); profileLogger.clearProfile(); } + public final void testFetchDenseVector() throws IOException { + Map response = esql(""" + | EVAL k = SCORE(v_l2_norm(f_dense_vector, [1])) + | KEEP _index, f_dense_vector + | LIMIT 1000 + """); + List columns = (List) response.get("columns"); + List values = (List) response.get("values"); + + MapMatcher expectedColumns = matchesMap().entry("f_dense_vector", "dense_vector").entry("_index", "keyword"); + assertMap(nameToType(columns), expectedColumns); + + MapMatcher expectedAllValues = matchesMap(); + for (Map.Entry e : allNodeToInfo().entrySet()) { + String nodeName = e.getKey(); + NodeInfo nodeInfo = e.getValue(); + MapMatcher expectedValues = matchesMap(); + expectedValues = expectedValues.entry("f_dense_vector", expectedDenseVector(nodeInfo.version)); + expectedValues = expectedValues.entry("_index", expectedIndex(nodeName, nodeInfo)); + expectedAllValues = expectedAllValues.entry(expectedIndex(nodeName, nodeInfo), expectedValues); + } + assertMap(indexToRow(columns, values), expectedAllValues); + } + + private Map esql(String query) throws IOException { + Request request = new Request("POST", "_query"); + XContentBuilder body = JsonXContent.contentBuilder().startObject(); + body.field("query", "FROM *:%mode%*,%mode%* METADATA _index".replace("%mode%", indexMode.toString()) + query); + { + body.startObject("pragma"); + if (extractPreference != null) { + body.field("field_extract_preference", extractPreference); + } + body.endObject(); + } + body.field("accept_pragma_risks", "true"); + body.field("profile", true); + body.field("include_ccs_metadata", true); + body.endObject(); + request.setJsonEntity(Strings.toString(body)); + + Map response = responseAsMap(client().performRequest(request)); + profileLogger.extractProfile(response, true); + if ((Boolean) response.get("is_partial")) { + throw new AssertionError("partial results: " + response); + } + return response; + } + protected void createIndexForNode(RestClient client, String nodeName, String nodeId) throws IOException { String indexName = indexMode + "_" + nodeName; if (false == indexExists(client, indexName)) { @@ -330,15 +354,18 @@ private Matcher expectedValue(Version minVersion, Version version, DataType t if (minVersion.onOrAfter(Version.V_9_2_0)) { yield nullValue(); } - Matcher> expected = version.onOrAfter(Version.V_9_2_0) - ? matchesList().item(0.5).item(10.0).item(5.9999995) - : matchesList().item(0.04283529).item(0.85670584).item(0.5140235); - yield anyOf(nullValue(), expected); + yield anyOf(nullValue(), expectedDenseVector(version)); } default -> throw new AssertionError("unsupported field type [" + type + "]"); }; } + private Matcher> expectedDenseVector(Version version) { + return version.onOrAfter(Version.V_9_2_0) + ? matchesList().item(0.5).item(10.0).item(5.9999995) + : matchesList().item(0.04283529).item(0.85670584).item(0.5140235); + } + /** * Is the type supported in indices? */ @@ -377,6 +404,9 @@ private List names(List columns) { private Map> indexToRow(List columns, List values) { List names = names(columns); int timestampIdx = names.indexOf("_index"); + if (timestampIdx < 0) { + throw new IllegalStateException("query didn't return _index"); + } Map> result = new TreeMap<>(); for (Object r : values) { List row = (List) r; @@ -426,4 +456,12 @@ private boolean syntheticSourceByDefault() { case STANDARD, LOOKUP -> false; }; } + + private String expectedIndex(String nodeName, NodeInfo nodeInfo) { + String expectedIndex = indexMode + "_" + nodeName; + if (nodeInfo.cluster == null) { + return expectedIndex; + } + return nodeInfo.cluster + ":" + expectedIndex; + } } From 663c15ac908171ba78ac8d8257e574baabc552d7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 11:54:29 -0400 Subject: [PATCH 17/38] Integ tests --- .../qa/rest/AllSupportedFieldsTestCase.java | 34 +++++++++++++++---- .../xpack/esql/DenseVectorFieldTypeIT.java | 4 +++ .../xpack/esql/action/LookupJoinTypesIT.java | 3 +- .../xpack/esql/plugin/KnnFunctionIT.java | 10 +++--- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 18b1ba417ffba..4a7aac1d5da79 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -9,9 +9,11 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexMode; @@ -42,6 +44,7 @@ import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.any; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -139,7 +142,7 @@ public final void testFetchAll() throws IOException { List values = (List) response.get("values"); MapMatcher expectedColumns = matchesMap(); - Version minVersion = allNodeToInfo().values().stream().map(n -> n.version).min(Comparator.comparing(v -> v.id)).get(); + Version minVersion = minVersion(); for (DataType type : DataType.values()) { if (supportedInIndex(type) == false) { continue; @@ -180,11 +183,26 @@ public final void testFetchAll() throws IOException { } public final void testFetchDenseVector() throws IOException { - Map response = esql(""" - | EVAL k = SCORE(v_l2_norm(f_dense_vector, [1])) - | KEEP _index, f_dense_vector - | LIMIT 1000 - """); + Map response; + try { + response = esql(""" + | EVAL k = SCORE(v_l2_norm(f_dense_vector, [1])) + | KEEP _index, f_dense_vector + | LIMIT 1000 + """); + } catch (ResponseException e) { + Version minVersion = minVersion(); + if (minVersion.onOrAfter(Version.V_9_2_0)) { + throw new AssertionError("versions on or after 9.2.0 should support correctly fetch the dense_vector", e); + } + assertThat( + "old version should fail with this error", + EntityUtils.toString(e.getResponse().getEntity()), + containsString("Unknown function [v_l2_norm]") + ); + // Failure is expected and fine + return; + } List columns = (List) response.get("columns"); List values = (List) response.get("values"); @@ -464,4 +482,8 @@ private String expectedIndex(String nodeName, NodeInfo nodeInfo) { } return nodeInfo.cluster + ":" + expectedIndex; } + + private Version minVersion() throws IOException { + return allNodeToInfo().values().stream().map(n -> n.version).min(Comparator.comparing(v -> v.id)).get(); + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java index 4af049bbb7927..c9ee082a93740 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java @@ -93,6 +93,8 @@ public DenseVectorFieldTypeIT( public void testRetrieveFieldType() { var query = """ FROM test + | EVAL k = v_l2_norm(vector, [1]) + | DROP k """; try (var resp = run(query)) { @@ -105,6 +107,7 @@ public void testRetrieveFieldType() { public void testRetrieveTopNDenseVectorFieldData() { var query = """ FROM test + | EVAL k = v_l2_norm(vector, [1]) | KEEP id, vector | SORT id ASC """; @@ -132,6 +135,7 @@ public void testRetrieveTopNDenseVectorFieldData() { public void testRetrieveDenseVectorFieldData() { var query = """ FROM test + | EVAL k = v_l2_norm(vector, [1]) | KEEP id, vector """; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index 4c6951ce8e017..5c7d56511b6e5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -244,7 +244,8 @@ public LookupJoinTypesIT(BinaryComparisonOperation operation) { if (type == NULL || type == DOC_DATA_TYPE || type == TSID_DATA_TYPE - || type == AGGREGATE_METRIC_DOUBLE + || type == AGGREGATE_METRIC_DOUBLE // need special handling for loads at the moment + || type == DENSE_VECTOR // need special handling for loads at the moment || type == GEOHASH || type == GEOTILE || type == GEOHEX diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java index c9efb5c3ae409..c8a426113eaf6 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java @@ -220,10 +220,12 @@ public void testKnnWithLookupJoin() { var error = expectThrows(VerificationException.class, () -> run(query)); assertThat( error.getMessage(), - containsString( - "line 3:13: [KNN] function cannot operate on [lookup_vector], supplied by an index [test_lookup] in non-STANDARD " - + "mode [lookup]" - ) + // TODO revert this when we have proper versioned type resolutions + // containsString( + // "line 3:13: [KNN] function cannot operate on [lookup_vector], supplied by an index [test_lookup] in non-STANDARD " + // + "mode [lookup]" + // ) + containsString("line 3:13: Cannot use field [lookup_vector] with unsupported type [dense_vector]") ); } From 1da11c74cf8776d7c79a3486771caebda89b7fe9 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 11:55:11 -0400 Subject: [PATCH 18/38] to_dense_Vector --- .../java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java index dc0e1f9836c1a..53460c3986823 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java @@ -64,6 +64,7 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) { Holder supportsDenseVector = new Holder<>(false); plan.forEachDown(p -> p.forEachExpression(UnresolvedFunction.class, fn -> { if (fn.name().equalsIgnoreCase("knn") + || fn.name().equalsIgnoreCase("to_dense_vector") || fn.name().equalsIgnoreCase("v_cosine") || fn.name().equalsIgnoreCase("v_hamming") || fn.name().equalsIgnoreCase("v_l1_norm") From a76afc12b90910ce235ce0217ba06d3bea193521 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 12:01:24 -0400 Subject: [PATCH 19/38] Drop score --- .../esql/qa/rest/generative/GenerativeForkRestTest.java | 4 ---- .../src/main/resources/dense_vector-bit.csv-spec | 4 ++-- .../src/main/resources/dense_vector-byte.csv-spec | 6 +++--- .../testFixtures/src/main/resources/dense_vector.csv-spec | 6 +++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java index e23ecebc99c56..5b2ea9a525489 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java @@ -54,10 +54,6 @@ protected void shouldSkipTest(String testName) throws IOException { "Tests using INSIST are not supported for now", testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName()) ); - assumeFalse( - "NOCOMMIT decide if we should keep this", - testCase.requiredCapabilities.contains(DENSE_VECTOR_FIELD_TYPE_BIT_ELEMENTS.capabilityName()) - ); assumeTrue("Cluster needs to support FORK", hasCapabilities(adminClient(), List.of(FORK_V9.capabilityName()))); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec index 9ddfe81b405df..801533229ac21 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec @@ -2,7 +2,7 @@ retrieveBitVectorData required_capability: dense_vector_field_type_bit_elements FROM dense_vector -| EVAL k = SCORE(v_l2_norm(bit_vector, [1])) +| EVAL k = v_l2_norm(bit_vector, [1]) | KEEP id, bit_vector | SORT id ; @@ -35,7 +35,7 @@ required_capability: dense_vector_field_type_bit_elements FROM dense_vector | EVAL v = bit_vector -| EVAL k = SCORE(v_l2_norm(bit_vector, [1])) +| EVAL k = v_l2_norm(bit_vector, [1]) | RENAME v AS new_vector | DROP float_vector, byte_vector, bit_vector, k | SORT id diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec index 3767b899b5214..14f3046f534bc 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec @@ -2,7 +2,7 @@ retrieveByteVectorData required_capability: dense_vector_field_type_byte_elements FROM dense_vector -| EVAL k = SCORE(v_l2_norm(byte_vector, [1])) +| EVAL k = v_l2_norm(byte_vector, [1]) | KEEP id, byte_vector | SORT id ; @@ -19,7 +19,7 @@ required_capability: dense_vector_field_type_byte_elements FROM dense_vector | EVAL v = byte_vector -| EVAL k = SCORE(v_l2_norm(byte_vector, [1])) +| EVAL k = v_l2_norm(byte_vector, [1]) | KEEP id, v | SORT id ; @@ -36,7 +36,7 @@ required_capability: dense_vector_field_type_byte_elements FROM dense_vector | EVAL v = byte_vector -| EVAL k = SCORE(v_l2_norm(byte_vector, [1])) +| EVAL k = v_l2_norm(byte_vector, [1]) | RENAME v AS new_vector | DROP float_vector, byte_vector, bit_vector, k | SORT id diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec index 4f0d7fd60f248..211c0784e962f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec @@ -2,7 +2,7 @@ retrieveDenseVectorData required_capability: dense_vector_field_type FROM dense_vector -| EVAL k = SCORE(v_l2_norm(float_vector, [1])) +| EVAL k = v_l2_norm(float_vector, [1]) | KEEP id, float_vector | SORT id ; @@ -19,7 +19,7 @@ required_capability: dense_vector_field_type FROM dense_vector | EVAL v = float_vector -| EVAL k = SCORE(v_l2_norm(float_vector, [1])) +| EVAL k = v_l2_norm(float_vector, [1]) | KEEP id, v | SORT id ; @@ -36,7 +36,7 @@ required_capability: dense_vector_field_type FROM dense_vector | EVAL v = float_vector -| EVAL k = SCORE(v_l2_norm(float_vector, [1])) +| EVAL k = v_l2_norm(float_vector, [1]) | RENAME v AS new_vector | DROP float_vector, byte_vector, bit_vector, k | SORT id From 1e8c58b8028458a41b8b421c21e0dd132757917d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 12:05:03 -0400 Subject: [PATCH 20/38] More score --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 2 +- .../testFixtures/src/main/resources/dense_vector-bit.csv-spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 4a7aac1d5da79..0fa023a3b6c04 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -186,7 +186,7 @@ public final void testFetchDenseVector() throws IOException { Map response; try { response = esql(""" - | EVAL k = SCORE(v_l2_norm(f_dense_vector, [1])) + | EVAL k = v_l2_norm(f_dense_vector, [1]) | KEEP _index, f_dense_vector | LIMIT 1000 """); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec index 801533229ac21..15fb805a14ef8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec @@ -18,7 +18,7 @@ denseBitVectorWithEval required_capability: dense_vector_field_type_bit_elements FROM dense_vector -| EVAL v = bit_vector, k = SCORE(v_l2_norm(bit_vector, [1])) +| EVAL v = bit_vector, k = v_l2_norm(bit_vector, [1]) | KEEP id, v | SORT id ; From 90e21a06d8ed100c36203eb3f084692517d2b046 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 12:18:20 -0400 Subject: [PATCH 21/38] Move created version --- .../xpack/esql/core/type/CreatedVersion.java | 44 +++++++++++++++++++ .../xpack/esql/core/type/DataType.java | 9 ++-- 2 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/CreatedVersion.java diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/CreatedVersion.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/CreatedVersion.java new file mode 100644 index 0000000000000..3f775a1bf20fa --- /dev/null +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/CreatedVersion.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.type; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.StreamOutput; + +/** + * Version that supports a {@link DataType}. + */ +interface CreatedVersion { + boolean supports(StreamOutput out); + + CreatedVersion SUPPORTED_ON_ALL_NODES = new CreatedVersion() { + @Override + public boolean supports(StreamOutput out) { + return true; + } + + @Override + public String toString() { + return "SupportedOnAllVersions"; + } + }; + + static CreatedVersion supportedOn(TransportVersion createdVersion) { + return new CreatedVersion() { + @Override + public boolean supports(StreamOutput out) { + return out.getTransportVersion().supports(createdVersion); + } + + @Override + public String toString() { + return "SupportedOn[" + createdVersion + "]"; + } + }; + } +} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index c4e42524ef661..6afedf997ba25 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -34,7 +34,6 @@ import java.util.function.Function; import static java.util.stream.Collectors.toMap; -import static org.elasticsearch.TransportVersions.INDEXING_PRESSURE_THROTTLING_STATS; import static org.elasticsearch.TransportVersions.INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING; import static org.elasticsearch.TransportVersions.ML_INFERENCE_SAGEMAKER_CHAT_COMPLETION; @@ -397,7 +396,7 @@ public enum DataType implements Writeable { /** * Version that first created this data type. */ - private final TransportVersion createdVersion; + private final CreatedVersion createdVersion; DataType(Builder builder) { String typeString = builder.typeName != null ? builder.typeName : builder.esType; @@ -754,7 +753,7 @@ public DataType counter() { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().supports(createdVersion) == false) { + if (createdVersion.supports(out) == false) { /* * TODO when we implement version aware planning flip this to an IllegalStateException * so we throw a 500 error. It'll be our bug then. Right now it's a sign that the user @@ -889,7 +888,7 @@ private static class Builder { * version for which we maintain wire compatibility, which is pretty * much {@code 8.18.0}. */ - private TransportVersion createdVersion = INDEXING_PRESSURE_THROTTLING_STATS; + private CreatedVersion createdVersion = CreatedVersion.SUPPORTED_ON_ALL_NODES; Builder() {} @@ -948,7 +947,7 @@ Builder counter(DataType counter) { } Builder createdVersion(TransportVersion createdVersion) { - this.createdVersion = createdVersion; + this.createdVersion = CreatedVersion.supportedOn(createdVersion); return this; } } From c30668ffb7f1ccd0013d15f22763d7d5f941e249 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 14:31:22 -0400 Subject: [PATCH 22/38] Test better --- .../xpack/esql/core/type/CreatedVersion.java | 11 ++- .../xpack/esql/core/type/DataType.java | 6 +- .../qa/rest/AllSupportedFieldsTestCase.java | 78 +++++++++++-------- 3 files changed, 56 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/CreatedVersion.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/CreatedVersion.java index 3f775a1bf20fa..401e8e3d8197c 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/CreatedVersion.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/CreatedVersion.java @@ -8,17 +8,16 @@ package org.elasticsearch.xpack.esql.core.type; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.StreamOutput; /** * Version that supports a {@link DataType}. */ -interface CreatedVersion { - boolean supports(StreamOutput out); +public interface CreatedVersion { + boolean supports(TransportVersion version); CreatedVersion SUPPORTED_ON_ALL_NODES = new CreatedVersion() { @Override - public boolean supports(StreamOutput out) { + public boolean supports(TransportVersion version) { return true; } @@ -31,8 +30,8 @@ public String toString() { static CreatedVersion supportedOn(TransportVersion createdVersion) { return new CreatedVersion() { @Override - public boolean supports(StreamOutput out) { - return out.getTransportVersion().supports(createdVersion); + public boolean supports(TransportVersion version) { + return version.supports(createdVersion); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 6afedf997ba25..be96a95d6710d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -753,7 +753,7 @@ public DataType counter() { @Override public void writeTo(StreamOutput out) throws IOException { - if (createdVersion.supports(out) == false) { + if (createdVersion.supports(out.getTransportVersion()) == false) { /* * TODO when we implement version aware planning flip this to an IllegalStateException * so we throw a 500 error. It'll be our bug then. Right now it's a sign that the user @@ -816,6 +816,10 @@ public boolean isDate() { }; } + public CreatedVersion createdVersion() { + return createdVersion; + } + public static DataType suggestedCast(Set originalTypes) { if (originalTypes.isEmpty() || originalTypes.contains(UNSUPPORTED)) { return null; diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 0fa023a3b6c04..d8dbf22073d06 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -10,9 +10,8 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.http.util.EntityUtils; -import org.elasticsearch.Version; +import org.elasticsearch.TransportVersion; import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; @@ -29,9 +28,7 @@ import org.junit.Before; import org.junit.Rule; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -39,6 +36,8 @@ import java.util.Map; import java.util.TreeMap; +import static org.elasticsearch.TransportVersions.NEW_SEMANTIC_QUERY_INTERCEPTORS; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; @@ -91,7 +90,7 @@ protected AllSupportedFieldsTestCase(MappedFieldType.FieldExtractPreference extr this.indexMode = indexMode; } - protected record NodeInfo(String cluster, String id, Version version) {} + protected record NodeInfo(String cluster, String id, TransportVersion version) {} private static Map nodeToInfo; @@ -111,18 +110,17 @@ protected Map allNodeToInfo() throws IOException { protected static Map fetchNodeToInfo(RestClient client, String cluster) throws IOException { Map nodeToInfo = new TreeMap<>(); - Request getIds = new Request("GET", "_cat/nodes"); - getIds.addParameter("h", "name,id,version"); - getIds.addParameter("full_id", ""); - Response idsResponse = client.performRequest(getIds); - BufferedReader idsReader = new BufferedReader(new InputStreamReader(idsResponse.getEntity().getContent())); - String line; - while ((line = idsReader.readLine()) != null) { - logger.info("node: {}", line); - String[] l = line.split(" "); - // TODO what's the right thing to use instead of Version? - nodeToInfo.put(l[0], new NodeInfo(cluster, l[1], Version.fromString(l[2]))); + Request request = new Request("GET", "/_nodes"); + Map response = responseAsMap(client.performRequest(request)); + Map nodes = (Map) extractValue(response, "nodes"); + for (Map.Entry n : nodes.entrySet()) { + String id = (String) n.getKey(); + Map nodeInfo = (Map) n.getValue(); + String nodeName = (String) extractValue(nodeInfo, "name"); + TransportVersion transportVersion = TransportVersion.fromId((Integer) extractValue(nodeInfo, "transport_version")); + nodeToInfo.put(nodeName, new NodeInfo(cluster, id, transportVersion)); } + return nodeToInfo; } @@ -138,11 +136,14 @@ public final void testFetchAll() throws IOException { , _id, _ignored, _index_mode, _score, _source, _version | LIMIT 1000 """); + if ((Boolean) response.get("is_partial")) { + throw new AssertionError("partial results: " + response); + } List columns = (List) response.get("columns"); List values = (List) response.get("values"); MapMatcher expectedColumns = matchesMap(); - Version minVersion = minVersion(); + TransportVersion minVersion = minVersion(); for (DataType type : DataType.values()) { if (supportedInIndex(type) == false) { continue; @@ -190,15 +191,31 @@ public final void testFetchDenseVector() throws IOException { | KEEP _index, f_dense_vector | LIMIT 1000 """); + if ((Boolean) response.get("is_partial")) { + TransportVersion minVersion = minVersion(); + Map clusters = (Map) response.get("_clusters"); + Map details = (Map) clusters.get("details"); + for (Map.Entry cluster : details.entrySet()) { + String failures = cluster.getValue().toString(); + if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { + throw new AssertionError("versions on or after 9.2.0 should support correctly fetch the dense_vector: " + failures); + } + assertThat(failures, containsString("doesn't understand data type [DENSE_VECTOR]")); + } + return; + } } catch (ResponseException e) { - Version minVersion = minVersion(); - if (minVersion.onOrAfter(Version.V_9_2_0)) { + TransportVersion minVersion = minVersion(); + if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { throw new AssertionError("versions on or after 9.2.0 should support correctly fetch the dense_vector", e); } assertThat( "old version should fail with this error", EntityUtils.toString(e.getResponse().getEntity()), - containsString("Unknown function [v_l2_norm]") + anyOf( + containsString("Unknown function [v_l2_norm]"), + containsString("Cannot use field [f_dense_vector] with unsupported type") + ) ); // Failure is expected and fine return; @@ -240,9 +257,6 @@ private Map esql(String query) throws IOException { Map response = responseAsMap(client().performRequest(request)); profileLogger.extractProfile(response, true); - if ((Boolean) response.get("is_partial")) { - throw new AssertionError("partial results: " + response); - } return response; } @@ -343,7 +357,7 @@ private void createAllTypesDoc(RestClient client, String indexName) throws IOExc client.performRequest(request); } - private Matcher expectedValue(Version minVersion, Version version, DataType type) { + private Matcher expectedValue(TransportVersion minVersion, TransportVersion version, DataType type) { return switch (type) { case BOOLEAN -> equalTo(true); case COUNTER_LONG, LONG, COUNTER_INTEGER, INTEGER, UNSIGNED_LONG, SHORT, BYTE -> equalTo(1); @@ -362,14 +376,14 @@ private Matcher expectedValue(Version minVersion, Version version, DataType t case GEO_SHAPE -> equalTo("POINT (-71.34 41.12)"); case NULL -> nullValue(); case AGGREGATE_METRIC_DOUBLE -> { - if (minVersion.onOrAfter(Version.V_9_2_0)) { + if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { yield nullValue(); } Matcher expected = equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); yield anyOf(nullValue(), expected); } case DENSE_VECTOR -> { - if (minVersion.onOrAfter(Version.V_9_2_0)) { + if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { yield nullValue(); } yield anyOf(nullValue(), expectedDenseVector(version)); @@ -378,8 +392,8 @@ private Matcher expectedValue(Version minVersion, Version version, DataType t }; } - private Matcher> expectedDenseVector(Version version) { - return version.onOrAfter(Version.V_9_2_0) + private Matcher> expectedDenseVector(TransportVersion version) { + return version.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS) ? matchesList().item(0.5).item(10.0).item(5.9999995) : matchesList().item(0.04283529).item(0.85670584).item(0.5140235); } @@ -441,7 +455,7 @@ private Map nameToValue(List names, List values) { return result; } - private Matcher expectedType(Version minVersion, DataType type) { + private Matcher expectedType(TransportVersion minVersion, DataType type) { return switch (type) { case COUNTER_DOUBLE, COUNTER_LONG, COUNTER_INTEGER -> { if (indexMode == IndexMode.TIME_SERIES) { @@ -454,7 +468,7 @@ private Matcher expectedType(Version minVersion, DataType type) { case NULL -> equalTo("keyword"); // Currently unsupported without TS command or KNN function case AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> { - if (false == minVersion.onOrAfter(Version.V_9_2_0)) { + if (false == minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { yield either(equalTo("unsupported")).or(equalTo(type.esType())); } yield equalTo("unsupported"); @@ -483,7 +497,7 @@ private String expectedIndex(String nodeName, NodeInfo nodeInfo) { return nodeInfo.cluster + ":" + expectedIndex; } - private Version minVersion() throws IOException { - return allNodeToInfo().values().stream().map(n -> n.version).min(Comparator.comparing(v -> v.id)).get(); + private TransportVersion minVersion() throws IOException { + return allNodeToInfo().values().stream().map(n -> n.version).min(Comparator.naturalOrder()).get(); } } From 77b59d4deda0c799fe12722ffe7a4327a1b00a44 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 14:37:41 -0400 Subject: [PATCH 23/38] More explain --- .../xpack/esql/qa/mixed/AllSupportedFieldsIT.java | 3 +++ .../org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java | 3 +++ .../xpack/esql/qa/single_node/AllSupportedFieldsIT.java | 3 +++ 3 files changed, 9 insertions(+) diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java index 8901f8aa70ea5..2120f3b2bcaac 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/AllSupportedFieldsIT.java @@ -16,6 +16,9 @@ import org.elasticsearch.xpack.esql.qa.rest.AllSupportedFieldsTestCase; import org.junit.ClassRule; +/** + * Fetch all field types in a mixed version cluster, simulating a rolling upgrade. + */ @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class AllSupportedFieldsIT extends AllSupportedFieldsTestCase { @ClassRule diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java index 42f23f9a3f0d7..ae1fa05c88555 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java @@ -27,6 +27,9 @@ import java.util.Map; import java.util.TreeMap; +/** + * Fetch all field types via cross cluster search, possible on a different version. + */ @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class AllSupportedFieldsIT extends AllSupportedFieldsTestCase { static ElasticsearchCluster remoteCluster = Clusters.remoteCluster(); diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AllSupportedFieldsIT.java index dddc806d001f8..aaa2be1750be4 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AllSupportedFieldsIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AllSupportedFieldsIT.java @@ -16,6 +16,9 @@ import org.elasticsearch.xpack.esql.qa.rest.AllSupportedFieldsTestCase; import org.junit.ClassRule; +/** + * Simple test for fetching all supported field types. + */ @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class AllSupportedFieldsIT extends AllSupportedFieldsTestCase { @ClassRule From a51592417448ce7b697fa9240b9a76d8f52eeb4f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 19:33:39 -0400 Subject: [PATCH 24/38] Comments --- .../qa/rest/AllSupportedFieldsTestCase.java | 2 +- .../main/resources/dense_vector-bit.csv-spec | 6 ++-- .../main/resources/dense_vector-byte.csv-spec | 6 ++-- .../src/main/resources/dense_vector.csv-spec | 6 ++-- .../xpack/esql/DenseVectorFieldTypeIT.java | 6 ++-- .../xpack/esql/analysis/PreAnalyzer.java | 9 ++++++ .../esql/enrich/EnrichPolicyResolver.java | 1 + .../xpack/esql/session/EsqlSession.java | 1 + .../rest-api-spec/test/esql/40_tsdb.yml | 28 +++++++++---------- .../rest-api-spec/test/esql/46_downsample.yml | 10 +++---- 10 files changed, 43 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index d8dbf22073d06..b23c4bb03d7d1 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -187,7 +187,7 @@ public final void testFetchDenseVector() throws IOException { Map response; try { response = esql(""" - | EVAL k = v_l2_norm(f_dense_vector, [1]) + | EVAL k = v_l2_norm(f_dense_vector, [1]) // workaround to enable fetching dense_vector | KEEP _index, f_dense_vector | LIMIT 1000 """); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec index 15fb805a14ef8..0b59dff14d47e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec @@ -2,7 +2,7 @@ retrieveBitVectorData required_capability: dense_vector_field_type_bit_elements FROM dense_vector -| EVAL k = v_l2_norm(bit_vector, [1]) +| EVAL k = v_l2_norm(bit_vector, [1]) // workaround to enable fetching dense_vector | KEEP id, bit_vector | SORT id ; @@ -18,7 +18,7 @@ denseBitVectorWithEval required_capability: dense_vector_field_type_bit_elements FROM dense_vector -| EVAL v = bit_vector, k = v_l2_norm(bit_vector, [1]) +| EVAL v = bit_vector, k = v_l2_norm(bit_vector, [1]) // workaround to enable fetching dense_vector | KEEP id, v | SORT id ; @@ -35,7 +35,7 @@ required_capability: dense_vector_field_type_bit_elements FROM dense_vector | EVAL v = bit_vector -| EVAL k = v_l2_norm(bit_vector, [1]) +| EVAL k = v_l2_norm(bit_vector, [1]) // workaround to enable fetching dense_vector | RENAME v AS new_vector | DROP float_vector, byte_vector, bit_vector, k | SORT id diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec index 14f3046f534bc..588fa704504c1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec @@ -2,7 +2,7 @@ retrieveByteVectorData required_capability: dense_vector_field_type_byte_elements FROM dense_vector -| EVAL k = v_l2_norm(byte_vector, [1]) +| EVAL k = v_l2_norm(byte_vector, [1]) // workaround to enable fetching dense_vector | KEEP id, byte_vector | SORT id ; @@ -19,7 +19,7 @@ required_capability: dense_vector_field_type_byte_elements FROM dense_vector | EVAL v = byte_vector -| EVAL k = v_l2_norm(byte_vector, [1]) +| EVAL k = v_l2_norm(byte_vector, [1]) // workaround to enable fetching dense_vector | KEEP id, v | SORT id ; @@ -36,7 +36,7 @@ required_capability: dense_vector_field_type_byte_elements FROM dense_vector | EVAL v = byte_vector -| EVAL k = v_l2_norm(byte_vector, [1]) +| EVAL k = v_l2_norm(byte_vector, [1]) // workaround to enable fetching dense_vector | RENAME v AS new_vector | DROP float_vector, byte_vector, bit_vector, k | SORT id diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec index 211c0784e962f..ac75f2d82a95f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec @@ -2,7 +2,7 @@ retrieveDenseVectorData required_capability: dense_vector_field_type FROM dense_vector -| EVAL k = v_l2_norm(float_vector, [1]) +| EVAL k = v_l2_norm(float_vector, [1]) // workaround to enable fetching dense_vector | KEEP id, float_vector | SORT id ; @@ -19,7 +19,7 @@ required_capability: dense_vector_field_type FROM dense_vector | EVAL v = float_vector -| EVAL k = v_l2_norm(float_vector, [1]) +| EVAL k = v_l2_norm(float_vector, [1]) // workaround to enable fetching dense_vector | KEEP id, v | SORT id ; @@ -36,7 +36,7 @@ required_capability: dense_vector_field_type FROM dense_vector | EVAL v = float_vector -| EVAL k = v_l2_norm(float_vector, [1]) +| EVAL k = v_l2_norm(float_vector, [1]) // workaround to enable fetching dense_vector | RENAME v AS new_vector | DROP float_vector, byte_vector, bit_vector, k | SORT id diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java index c9ee082a93740..3014e1d4ba48a 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java @@ -93,7 +93,7 @@ public DenseVectorFieldTypeIT( public void testRetrieveFieldType() { var query = """ FROM test - | EVAL k = v_l2_norm(vector, [1]) + | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector | DROP k """; @@ -107,7 +107,7 @@ public void testRetrieveFieldType() { public void testRetrieveTopNDenseVectorFieldData() { var query = """ FROM test - | EVAL k = v_l2_norm(vector, [1]) + | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector | KEEP id, vector | SORT id ASC """; @@ -135,7 +135,7 @@ public void testRetrieveTopNDenseVectorFieldData() { public void testRetrieveDenseVectorFieldData() { var query = """ FROM test - | EVAL k = v_l2_norm(vector, [1]) + | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector | KEEP id, vector """; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java index 53460c3986823..fba3fa3b966c3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java @@ -60,6 +60,15 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) { List unresolvedEnriches = new ArrayList<>(); plan.forEachUp(Enrich.class, unresolvedEnriches::add); + /* + * Enable aggregate_metric_double and dense_vector when we see certain function + * or the TS command. This allows us to release these when not all nodes understand + * these types. These functions are only supported on newer nodes, so we use them + * as a signal that the query is only for nodes that support these types. + * + * This work around is temporary until we flow the minimum transport version + * back through a cross cluster search field caps call. + */ Holder supportsAggregateMetricDouble = new Holder<>(false); Holder supportsDenseVector = new Holder<>(false); plan.forEachDown(p -> p.forEachExpression(UnresolvedFunction.class, fn -> { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java index b4a3402bfa62d..fa3fbd83ec414 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java @@ -450,6 +450,7 @@ public void messageReceived(LookupRequest request, TransportChannel channel, Tas IndexResolver.ALL_FIELDS, null, false, + // Disable aggregate_metric_double and dense_vector until we get version checks in planning false, false, refs.acquire(indexResult -> { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 5c629b508f380..d09f41a9f280e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -494,6 +494,7 @@ private void preAnalyzeLookupIndex( result.wildcardJoinIndices().contains(localPattern) ? IndexResolver.ALL_FIELDS : result.fieldNames, null, false, + // Disable aggregate_metric_double and dense_vector until we get version checks in planning false, false, listener.map(indexResolution -> receiveLookupIndexResolution(result, localPattern, executionInfo, indexResolution)) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index e93bd89bf7e1a..8a186d8881abf 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -233,7 +233,7 @@ filter on counter without cast: body: query: | FROM test - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | WHERE k8s.pod.network.tx == 1434577921 --- @@ -268,7 +268,7 @@ sort on counter without cast: body: query: | FROM test - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | KEEP k8s.pod.network.tx | SORT k8s.pod.network.tx | LIMIT 1 @@ -299,7 +299,7 @@ from doc with aggregate_metric_double: body: query: | FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | DROP a - match: {columns.0.name: "@timestamp"} @@ -331,7 +331,7 @@ stats on aggregate_metric_double: body: query: | FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) - length: {values: 1} - length: {values.0: 4} @@ -365,7 +365,7 @@ grouping stats on aggregate_metric_double: body: query: | FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY dim | SORT dim - length: {values: 2} @@ -408,7 +408,7 @@ sorting with aggregate_metric_double with partial submetrics: body: query: | FROM test3 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | SORT @timestamp | KEEP @timestamp, agg_metric @@ -443,7 +443,7 @@ aggregate_metric_double unsortable: body: query: | FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | DROP a | SORT agg_metric @@ -464,7 +464,7 @@ stats on aggregate_metric_double with partial submetrics: body: query: | FROM test3 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY k8s.pod.uid | SORT k8s.pod.uid @@ -508,7 +508,7 @@ stats on aggregate_metric_double missing min and max: body: query: | FROM test4 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) - length: {values: 1} @@ -543,7 +543,7 @@ render aggregate_metric_double when missing min and max: body: query: | FROM test4 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | KEEP agg_metric - length: {values: 1} @@ -570,7 +570,7 @@ render aggregate_metric_double when missing value: body: query: | FROM test3 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | WHERE @timestamp == "2021-04-28T19:51:04.467Z" | KEEP agg_metric @@ -598,7 +598,7 @@ to_string aggregate_metric_double: body: query: | FROM test4 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | EVAL agg = to_string(agg_metric) | KEEP agg @@ -625,7 +625,7 @@ from index pattern unsupported counter: body: query: | FROM test* - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | DROP a - match: {columns.0.name: "@timestamp"} @@ -771,7 +771,7 @@ avg of aggregate_metric_double: body: query: | FROM test2 - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | STATS avg = avg(agg_metric) | KEEP avg diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml index 95ff143de8de8..49f6248245451 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml @@ -100,7 +100,7 @@ setup: body: query: | FROM test-downsample - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | DROP a | STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx) | LIMIT 100 @@ -145,7 +145,7 @@ setup: body: query: | FROM test-downsample - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | DROP a | WHERE @timestamp == "2021-04-28T19:00:00.000Z" | KEEP k8s.pod.network.rx @@ -348,7 +348,7 @@ setup: body: query: | FROM test-* - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | WHERE k8s.pod.uid == \"947e4ced-1786-4e53-9e0c-5c447e959507\" | STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx), avg(k8s.pod.network.rx) | LIMIT 100 @@ -688,7 +688,7 @@ setup: body: query: | FROM test-* - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | DROP a | SORT some_field, @timestamp, k8s.pod.uid | KEEP k8s.pod.network.rx, some_field, @timestamp @@ -744,7 +744,7 @@ setup: body: query: | FROM test-downsample - | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) + | EVAL a = TO_AGGREGATE_METRIC_DOUBLE(1) // Temporary workaround to enable aggregate_metric_double | DROP a | MV_EXPAND k8s.pod.network.rx | SORT @timestamp, k8s.pod.uid From 8f2d210267b1b486c7155afbe418916bcfa34d5b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Sep 2025 21:23:54 -0400 Subject: [PATCH 25/38] ibwc --- .../qa/rest/AllSupportedFieldsTestCase.java | 9 +++-- .../main/resources/dense_vector-bit.csv-spec | 3 ++ .../main/resources/dense_vector-byte.csv-spec | 3 ++ .../src/main/resources/dense_vector.csv-spec | 3 ++ .../xpack/esql/action/EsqlCapabilities.java | 4 ++- .../rest-api-spec/test/esql/40_tsdb.yml | 34 ++++++++++++------- .../test/esql/40_unsupported_types.yml | 4 +-- .../rest-api-spec/test/esql/46_downsample.yml | 2 +- 8 files changed, 42 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index b23c4bb03d7d1..d3ed66f86baab 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -377,6 +377,7 @@ private Matcher expectedValue(TransportVersion minVersion, TransportVersion v case NULL -> nullValue(); case AGGREGATE_METRIC_DOUBLE -> { if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { + // If all versions a new we get null. If any are old, some *might* support aggregate_metric_double yield nullValue(); } Matcher expected = equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); @@ -384,6 +385,7 @@ private Matcher expectedValue(TransportVersion minVersion, TransportVersion v } case DENSE_VECTOR -> { if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { + // If all versions a new we get null. If any are old, some *might* support dense_vector yield nullValue(); } yield anyOf(nullValue(), expectedDenseVector(version)); @@ -468,10 +470,11 @@ private Matcher expectedType(TransportVersion minVersion, DataType type) case NULL -> equalTo("keyword"); // Currently unsupported without TS command or KNN function case AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> { - if (false == minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { - yield either(equalTo("unsupported")).or(equalTo(type.esType())); + if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { + // If all versions a new we get null. If any are old, some *might* support dense_vector + yield equalTo("unsupported"); } - yield equalTo("unsupported"); + yield either(equalTo("unsupported")).or(equalTo(type.esType())); } default -> equalTo(type.esType()); }; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec index 0b59dff14d47e..2db3e96de11c0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec @@ -1,5 +1,6 @@ retrieveBitVectorData required_capability: dense_vector_field_type_bit_elements +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL k = v_l2_norm(bit_vector, [1]) // workaround to enable fetching dense_vector @@ -16,6 +17,7 @@ id:l | bit_vector:dense_vector denseBitVectorWithEval required_capability: dense_vector_field_type_bit_elements +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = bit_vector, k = v_l2_norm(bit_vector, [1]) // workaround to enable fetching dense_vector @@ -32,6 +34,7 @@ id:l | v:dense_vector denseBitVectorWithRenameAndDrop required_capability: dense_vector_field_type_bit_elements +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = bit_vector diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec index 588fa704504c1..0ecf23332ad58 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec @@ -1,5 +1,6 @@ retrieveByteVectorData required_capability: dense_vector_field_type_byte_elements +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL k = v_l2_norm(byte_vector, [1]) // workaround to enable fetching dense_vector @@ -16,6 +17,7 @@ id:l | byte_vector:dense_vector denseByteVectorWithEval required_capability: dense_vector_field_type_byte_elements +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = byte_vector @@ -33,6 +35,7 @@ id:l | v:dense_vector denseByteVectorWithRenameAndDrop required_capability: dense_vector_field_type_byte_elements +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = byte_vector diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec index ac75f2d82a95f..b68fd9ba416b6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec @@ -1,5 +1,6 @@ retrieveDenseVectorData required_capability: dense_vector_field_type +required_capability: dense_vector_agg_metric_double_if_fns FROM dense_vector | EVAL k = v_l2_norm(float_vector, [1]) // workaround to enable fetching dense_vector @@ -16,6 +17,7 @@ id:l | float_vector:dense_vector denseVectorWithEval required_capability: dense_vector_field_type +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = float_vector @@ -33,6 +35,7 @@ id:l | v:dense_vector denseVectorWithRenameAndDrop required_capability: dense_vector_field_type +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL v = float_vector diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 68713c7a77701..43a8f5d3fe1e0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1544,7 +1544,9 @@ public enum Cap { */ TS_COMMAND_V0(), - FIX_ALIAS_ID_WHEN_DROP_ALL_AGGREGATES + FIX_ALIAS_ID_WHEN_DROP_ALL_AGGREGATES, + + DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS ; diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 8a186d8881abf..b9ec55a0c5223 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -238,6 +238,14 @@ filter on counter without cast: --- cast counter then filter: + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: POST + path: /_query + parameters: [ ] + capabilities: [ aggregate_metric_double_convert_to ] + reason: "Uses TO_AGGREGATE_METRIC_DOUBLE" - do: esql.query: body: @@ -260,7 +268,7 @@ sort on counter without cast: - method: POST path: /_query parameters: [] - capabilities: [sorting_on_source_and_counters_forbidden] + capabilities: [sorting_on_source_and_counters_forbidden, aggregate_metric_double_convert_to] reason: "Sorting on counters shouldn't have been possible" - do: catch: /cannot sort on counter_long/ @@ -290,7 +298,7 @@ from doc with aggregate_metric_double: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double] + capabilities: [aggregate_metric_double, aggregate_metric_double_convert_to] reason: "Support for aggregate_metric_double" - do: allowed_warnings_regex: @@ -322,7 +330,7 @@ stats on aggregate_metric_double: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double] + capabilities: [aggregate_metric_double, aggregate_metric_double_convert_to] reason: "Support for aggregate_metric_double" - do: allowed_warnings_regex: @@ -356,7 +364,7 @@ grouping stats on aggregate_metric_double: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double] + capabilities: [aggregate_metric_double, aggregate_metric_double_convert_to] reason: "Support for aggregate_metric_double" - do: allowed_warnings_regex: @@ -399,7 +407,7 @@ sorting with aggregate_metric_double with partial submetrics: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_sorting] + capabilities: [aggregate_metric_double_sorting, aggregate_metric_double_convert_to] reason: "Support for sorting when aggregate_metric_double present" - do: allowed_warnings_regex: @@ -435,7 +443,7 @@ aggregate_metric_double unsortable: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_sorting] + capabilities: [aggregate_metric_double_sorting, aggregate_metric_double_convert_to] reason: "Support for sorting when aggregate_metric_double present" - do: catch: /cannot sort on aggregate_metric_double/ @@ -455,7 +463,7 @@ stats on aggregate_metric_double with partial submetrics: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_partial_submetrics] + capabilities: [aggregate_metric_double_partial_submetrics, aggregate_metric_double_convert_to] reason: "Support for partial submetrics in aggregate_metric_double" - do: allowed_warnings_regex: @@ -499,7 +507,7 @@ stats on aggregate_metric_double missing min and max: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_partial_submetrics ] + capabilities: [ aggregate_metric_double_partial_submetrics, aggregate_metric_double_convert_to ] reason: "Support for partial submetrics in aggregate_metric_double" - do: allowed_warnings_regex: @@ -534,7 +542,7 @@ render aggregate_metric_double when missing min and max: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_rendering ] + capabilities: [ aggregate_metric_double_rendering, aggregate_metric_double_convert_to ] reason: "Support for rendering aggregate_metric_doubles" - do: allowed_warnings_regex: @@ -561,7 +569,7 @@ render aggregate_metric_double when missing value: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_rendering ] + capabilities: [ aggregate_metric_double_rendering, aggregate_metric_double_convert_to ] reason: "Support for rendering aggregate_metric_doubles" - do: allowed_warnings_regex: @@ -589,7 +597,7 @@ to_string aggregate_metric_double: - method: POST path: /_query parameters: [ ] - capabilities: [ aggregate_metric_double_rendering ] + capabilities: [ aggregate_metric_double_rendering, aggregate_metric_double_convert_to ] reason: "Support for rendering aggregate_metric_doubles" - do: allowed_warnings_regex: @@ -616,7 +624,7 @@ from index pattern unsupported counter: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_partial_submetrics] + capabilities: [aggregate_metric_double_partial_submetrics, aggregate_metric_double_convert_to] reason: "Support for partial submetrics in aggregate_metric_double" - do: allowed_warnings_regex: @@ -761,7 +769,7 @@ avg of aggregate_metric_double: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double_avg] + capabilities: [aggregate_metric_double_avg, aggregate_metric_double_convert_to] reason: "support avg aggregations with aggregate metric double" - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index cd6c4a1d0daaf..5b9249b262c7d 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -145,7 +145,7 @@ unsupported: - method: POST path: /_query parameters: [] - capabilities: [dense_vector_field_type] + capabilities: [dense_vector_field_type, dense_vector_agg_metric_double_if_fns] reason: "uses original_type" - do: @@ -338,7 +338,7 @@ unsupported with sort: - method: POST path: /_query parameters: [ ] - capabilities: [ dense_vector_field_type ] + capabilities: [ dense_vector_field_type, dense_vector_agg_metric_double_if_fns ] reason: "support for sorting when dense_vector_field_type present" - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml index 49f6248245451..a311889f8d492 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml @@ -83,7 +83,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double] + capabilities: [aggregate_metric_double, dense_vector_agg_metric_double_if_fns] reason: "Support for aggregate_metric_double" - do: indices.downsample: From 8b6c45f9430a5f1b336e6f7f5d772f1824408408 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 25 Sep 2025 09:41:41 -0400 Subject: [PATCH 26/38] Extra --- .../xpack/esql/analysis/AnalyzerTests.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index eaebf5c6109db..aa712f9f061f0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -136,6 +136,7 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.tsdbIndexResolution; import static org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; +import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; @@ -3355,6 +3356,30 @@ public void testResolveDenseVector() { } } + public void testResolveAggregateMetricDouble() { + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse( + "foo", + Map.of("v", new IndexFieldCapabilitiesBuilder("v", "aggregate_metric_double").build()) + ) + ), + List.of() + ); + { + IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); + var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); + assertThat(plan.output(), hasSize(1)); + assertThat(plan.output().getFirst().dataType(), equalTo(AGGREGATE_METRIC_DOUBLE)); + } + { + IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, false, true)); + var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); + assertThat(plan.output(), hasSize(1)); + assertThat(plan.output().getFirst().dataType(), equalTo(UNSUPPORTED)); + } + } + public void testBasicFork() { LogicalPlan plan = analyze(""" from test From c60a704ea1ba41d0629e697a80faba47063c7b41 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 25 Sep 2025 11:28:20 -0400 Subject: [PATCH 27/38] Udpate --- .../xpack/esql/ccq/AllSupportedFieldsIT.java | 9 ++++ .../qa/rest/AllSupportedFieldsTestCase.java | 53 +++++++++++-------- .../src/main/resources/enrich.csv-spec | 5 -- .../src/main/resources/lookup-join.csv-spec | 5 -- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java index ae1fa05c88555..b166e32a99b8d 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java @@ -24,6 +24,7 @@ import org.junit.rules.TestRule; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -88,4 +89,12 @@ public static void closeRemoteClient() throws IOException { remoteClient = null; } } + + @Override + protected boolean fetchDenseVectorAggMetricDoubleIfFns() throws IOException { + return super.fetchDenseVectorAggMetricDoubleIfFns() + && clusterHasCapability(remoteClient(), "GET", "/_query", List.of(), List.of("DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS")).orElse( + false + ); + } } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index d3ed66f86baab..4553013733ef6 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -101,6 +101,19 @@ private Map nodeToInfo() throws IOException { return nodeToInfo; } + private static Boolean denseVectorAggMetricDoubleIfFns; + + private boolean denseVectorAggMetricDoubleIfFns() throws IOException { + if (denseVectorAggMetricDoubleIfFns == null) { + denseVectorAggMetricDoubleIfFns = fetchDenseVectorAggMetricDoubleIfFns(); + } + return denseVectorAggMetricDoubleIfFns; + } + + protected boolean fetchDenseVectorAggMetricDoubleIfFns() throws IOException { + return clusterHasCapability("GET", "/_query", List.of(), List.of("DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS")).orElse(false); + } + /** * Map from node name to information about the node. */ @@ -143,12 +156,11 @@ public final void testFetchAll() throws IOException { List values = (List) response.get("values"); MapMatcher expectedColumns = matchesMap(); - TransportVersion minVersion = minVersion(); for (DataType type : DataType.values()) { if (supportedInIndex(type) == false) { continue; } - expectedColumns = expectedColumns.entry(fieldName(type), expectedType(minVersion, type)); + expectedColumns = expectedColumns.entry(fieldName(type), expectedType(type)); } expectedColumns = expectedColumns.entry("_id", "keyword") .entry("_ignored", "keyword") @@ -168,7 +180,7 @@ public final void testFetchAll() throws IOException { if (supportedInIndex(type) == false) { continue; } - expectedValues = expectedValues.entry(fieldName(type), expectedValue(minVersion, nodeInfo.version, type)); + expectedValues = expectedValues.entry(fieldName(type), expectedValue(nodeInfo.version, type)); } expectedValues = expectedValues.entry("_id", any(String.class)) .entry("_ignored", nullValue()) @@ -192,22 +204,23 @@ public final void testFetchDenseVector() throws IOException { | LIMIT 1000 """); if ((Boolean) response.get("is_partial")) { - TransportVersion minVersion = minVersion(); Map clusters = (Map) response.get("_clusters"); Map details = (Map) clusters.get("details"); + + boolean foundError = false; for (Map.Entry cluster : details.entrySet()) { String failures = cluster.getValue().toString(); - if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { - throw new AssertionError("versions on or after 9.2.0 should support correctly fetch the dense_vector: " + failures); + if (denseVectorAggMetricDoubleIfFns()) { + throw new AssertionError("should correctly fetch the dense_vector: " + failures); } - assertThat(failures, containsString("doesn't understand data type [DENSE_VECTOR]")); + foundError |= failures.contains("doesn't understand data type [DENSE_VECTOR]"); } + assertTrue("didn't find errors: " + details, foundError); return; } } catch (ResponseException e) { - TransportVersion minVersion = minVersion(); - if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { - throw new AssertionError("versions on or after 9.2.0 should support correctly fetch the dense_vector", e); + if (denseVectorAggMetricDoubleIfFns()) { + throw new AssertionError("should correctly fetch the dense_vector", e); } assertThat( "old version should fail with this error", @@ -357,7 +370,7 @@ private void createAllTypesDoc(RestClient client, String indexName) throws IOExc client.performRequest(request); } - private Matcher expectedValue(TransportVersion minVersion, TransportVersion version, DataType type) { + private Matcher expectedValue(TransportVersion version, DataType type) throws IOException { return switch (type) { case BOOLEAN -> equalTo(true); case COUNTER_LONG, LONG, COUNTER_INTEGER, INTEGER, UNSIGNED_LONG, SHORT, BYTE -> equalTo(1); @@ -376,16 +389,16 @@ private Matcher expectedValue(TransportVersion minVersion, TransportVersion v case GEO_SHAPE -> equalTo("POINT (-71.34 41.12)"); case NULL -> nullValue(); case AGGREGATE_METRIC_DOUBLE -> { - if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { - // If all versions a new we get null. If any are old, some *might* support aggregate_metric_double + if (denseVectorAggMetricDoubleIfFns()) { + // If all versions are new we get null. If any are old, some *might* support aggregate_metric_double yield nullValue(); } Matcher expected = equalTo("{\"min\":-302.5,\"max\":702.3,\"sum\":200.0,\"value_count\":25}"); yield anyOf(nullValue(), expected); } case DENSE_VECTOR -> { - if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { - // If all versions a new we get null. If any are old, some *might* support dense_vector + if (denseVectorAggMetricDoubleIfFns()) { + // If all versions are new we get null. If any are old, some *might* support dense_vector yield nullValue(); } yield anyOf(nullValue(), expectedDenseVector(version)); @@ -457,7 +470,7 @@ private Map nameToValue(List names, List values) { return result; } - private Matcher expectedType(TransportVersion minVersion, DataType type) { + private Matcher expectedType(DataType type) throws IOException { return switch (type) { case COUNTER_DOUBLE, COUNTER_LONG, COUNTER_INTEGER -> { if (indexMode == IndexMode.TIME_SERIES) { @@ -470,8 +483,8 @@ private Matcher expectedType(TransportVersion minVersion, DataType type) case NULL -> equalTo("keyword"); // Currently unsupported without TS command or KNN function case AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> { - if (minVersion.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS)) { - // If all versions a new we get null. If any are old, some *might* support dense_vector + if (denseVectorAggMetricDoubleIfFns()) { + // If all versions are new we get null. If any are old, some *might* support dense_vector yield equalTo("unsupported"); } yield either(equalTo("unsupported")).or(equalTo(type.esType())); @@ -499,8 +512,4 @@ private String expectedIndex(String nodeName, NodeInfo nodeInfo) { } return nodeInfo.cluster + ":" + expectedIndex; } - - private TransportVersion minVersion() throws IOException { - return allNodeToInfo().values().stream().map(n -> n.version).min(Comparator.naturalOrder()).get(); - } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec index 1075f474f823e..c263ce4326865 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec @@ -651,11 +651,6 @@ fieldsInOtherIndicesBug required_capability: enrich_load required_capability: fix_replace_missing_field_with_null_duplicate_name_id_in_layout -// from * accidentally selects columns with dense_vector field type. -// This is not properly handled when the query is planned by newer node and executed by an older one. -// see https://github.com/elastic/elasticsearch/issues/135193 -required_capability: dense_vector_field_type - from * | keep author.keyword, book_no, scalerank, street, bytes_in, @timestamp, abbrev, city_location, distance, description, birth_date, language_code, intersects, client_ip, event_duration, version | enrich languages_policy on author.keyword diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 7ca77696138a3..2aa88f47a0a96 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1674,11 +1674,6 @@ enrichLookupStatsBug required_capability: join_lookup_v12 required_capability: fix_replace_missing_field_with_null_duplicate_name_id_in_layout -// from * accidentally selects columns with dense_vector field type. -// This is not properly handled when the query is planned by newer node and executed by an older one. -// see https://github.com/elastic/elasticsearch/issues/135193 -required_capability: dense_vector_field_type - from * | enrich languages_policy on cluster | rename languages.byte as language_code From 78a85db985c40b74d15cd7edd034b9fc2668ca5a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 25 Sep 2025 16:03:23 +0000 Subject: [PATCH 28/38] [CI] Auto commit changes from spotless --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 4553013733ef6..f4414e0af4c10 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.TreeMap; From 9919907d6c39eb50056815b1405a24e3fc31a98b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 25 Sep 2025 14:31:20 -0400 Subject: [PATCH 29/38] mor efix --- x-pack/plugin/build.gradle | 13 +++++++++++++ .../xpack/esql/analysis/AnalyzerTests.java | 11 +++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 1e89582ba87e9..ea715b0d5c921 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -142,6 +142,19 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("ml/sparse_vector_search/Search on a sparse_vector field with dots in the field names", "Vectors are no longer returned by default") task.skipTest("ml/sparse_vector_search/Search on a nested sparse_vector field with dots in the field names and conflicting child fields", "Vectors are no longer returned by default") task.skipTest("esql/190_lookup_join/lookup-no-key-only-key", "Requires the fix") + task.skipTest("esql/40_tsdb/aggregate_metric_double unsortable", "Extra function required to enable the field type") + task.skipTest("esql/40_tsdb/avg of aggregate_metric_double", "Extra function required to enable the field type") + task.skipTest("esql/40_tsdb/grouping stats on aggregate_metric_double", "Extra function required to enable the field type") + task.skipTest("esql/40_tsdb/render aggregate_metric_double when missing min and max", "Extra function required to enable the field type") + task.skipTest("esql/40_tsdb/render aggregate_metric_double when missing value", "Extra function required to enable the field type") + task.skipTest("esql/40_tsdb/sorting with aggregate_metric_double with partial submetrics", "Extra function required to enable the field type") + task.skipTest("esql/40_tsdb/stats on aggregate_metric_double missing min and max", "Extra function required to enable the field type") + task.skipTest("esql/40_tsdb/to_string aggregate_metric_double", "Extra function required to enable the field type") + task.skipTest("esql/40_tsdb/stats on aggregate_metric_double with partial submetrics", "Extra function required to enable the field type") + task.skipTest("esql/46_downsample/MV_EXPAND on non-MV aggregate metric double", "Extra function required to enable the field type") + task.skipTest("esql/46_downsample/Query stats on downsampled index", "Extra function required to enable the field type") + task.skipTest("esql/46_downsample/Render stats from downsampled index", "Extra function required to enable the field type") + task.skipTest("esql/46_downsample/Sort from multiple indices one with aggregate metric double", "Extra function required to enable the field type") }) tasks.named('yamlRestCompatTest').configure { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 9c4630cdedd65..e664e50af835f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; +import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; @@ -3346,7 +3347,10 @@ public void testResolveDenseVector() { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); - assertThat(plan.output().getFirst().dataType(), equalTo(DENSE_VECTOR)); + assertThat(plan.output().getFirst().dataType(), equalTo( + DENSE_VECTOR_FEATURE_FLAG.isEnabled() ? + DENSE_VECTOR : UNSUPPORTED + )); } { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, false)); @@ -3370,7 +3374,10 @@ public void testResolveAggregateMetricDouble() { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); - assertThat(plan.output().getFirst().dataType(), equalTo(AGGREGATE_METRIC_DOUBLE)); + assertThat(plan.output().getFirst().dataType(), equalTo( + EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG.isEnabled() ? + AGGREGATE_METRIC_DOUBLE : UNSUPPORTED + )); } { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, false, true)); From 435d23379f97e3c0b8c741a8f005021e51301482 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 25 Sep 2025 14:37:52 -0400 Subject: [PATCH 30/38] Format --- .../xpack/esql/analysis/AnalyzerTests.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index e664e50af835f..fb7239b0c1f84 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3347,10 +3347,7 @@ public void testResolveDenseVector() { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); - assertThat(plan.output().getFirst().dataType(), equalTo( - DENSE_VECTOR_FEATURE_FLAG.isEnabled() ? - DENSE_VECTOR : UNSUPPORTED - )); + assertThat(plan.output().getFirst().dataType(), equalTo(DENSE_VECTOR_FEATURE_FLAG.isEnabled() ? DENSE_VECTOR : UNSUPPORTED)); } { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, false)); @@ -3374,10 +3371,10 @@ public void testResolveAggregateMetricDouble() { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); - assertThat(plan.output().getFirst().dataType(), equalTo( - EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG.isEnabled() ? - AGGREGATE_METRIC_DOUBLE : UNSUPPORTED - )); + assertThat( + plan.output().getFirst().dataType(), + equalTo(EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG.isEnabled() ? AGGREGATE_METRIC_DOUBLE : UNSUPPORTED) + ); } { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, false, true)); From 940a3b2bc978b131aadc7a8e838d736bf4147dbe Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 25 Sep 2025 15:47:45 -0400 Subject: [PATCH 31/38] Add other --- .../esql/qa/testFixtures/src/main/resources/enrich.csv-spec | 1 + .../esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec index c263ce4326865..1f8919fc7cd32 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec @@ -650,6 +650,7 @@ IDR | Indore | POINT(75.8472 22.7167) | India | POINT(75.8 fieldsInOtherIndicesBug required_capability: enrich_load required_capability: fix_replace_missing_field_with_null_duplicate_name_id_in_layout +required_capability: dense_vector_agg_metric_double_if_fns from * | keep author.keyword, book_no, scalerank, street, bytes_in, @timestamp, abbrev, city_location, distance, description, birth_date, language_code, intersects, client_ip, event_duration, version diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 2aa88f47a0a96..b0d6f1099ec1f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1673,6 +1673,7 @@ null |1952-02-27T00:00:00.000Z enrichLookupStatsBug required_capability: join_lookup_v12 required_capability: fix_replace_missing_field_with_null_duplicate_name_id_in_layout +required_capability: dense_vector_agg_metric_double_if_fns from * | enrich languages_policy on cluster From d4cfad839d838873ad2be76ecad2a1641d91bcbf Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 25 Sep 2025 16:01:24 -0400 Subject: [PATCH 32/38] Log --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index f4414e0af4c10..26ba9361eea01 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TreeMap; @@ -128,6 +129,7 @@ protected static Map fetchNodeToInfo(RestClient client, String for (Map.Entry n : nodes.entrySet()) { String id = (String) n.getKey(); Map nodeInfo = (Map) n.getValue(); + logger.error("DAFADFS {}", nodeInfo); String nodeName = (String) extractValue(nodeInfo, "name"); TransportVersion transportVersion = TransportVersion.fromId((Integer) extractValue(nodeInfo, "transport_version")); nodeToInfo.put(nodeName, new NodeInfo(cluster, id, transportVersion)); @@ -273,7 +275,7 @@ private Map esql(String query) throws IOException { } protected void createIndexForNode(RestClient client, String nodeName, String nodeId) throws IOException { - String indexName = indexMode + "_" + nodeName; + String indexName = indexMode + "_" + nodeName.toLowerCase(Locale.ROOT); if (false == indexExists(client, indexName)) { createAllTypesIndex(client, indexName, nodeId); createAllTypesDoc(client, indexName); @@ -505,7 +507,7 @@ private boolean syntheticSourceByDefault() { } private String expectedIndex(String nodeName, NodeInfo nodeInfo) { - String expectedIndex = indexMode + "_" + nodeName; + String expectedIndex = indexMode + "_" + nodeName.toLowerCase(Locale.ROOT); if (nodeInfo.cluster == null) { return expectedIndex; } From 94784ab7d1d09881e33d0db9de438020b1e20c62 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 25 Sep 2025 17:18:22 -0400 Subject: [PATCH 33/38] Some udpates --- .../xpack/esql/ccq/AllSupportedFieldsIT.java | 8 +- .../qa/rest/AllSupportedFieldsTestCase.java | 89 ++++++++++++++----- 2 files changed, 73 insertions(+), 24 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java index b166e32a99b8d..fc0763f7d0777 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/AllSupportedFieldsIT.java @@ -48,8 +48,12 @@ public AllSupportedFieldsIT(MappedFieldType.FieldExtractPreference extractPrefer @Before public void createRemoteIndices() throws IOException { - for (Map.Entry e : remoteNodeToInfo().entrySet()) { - createIndexForNode(remoteClient(), e.getKey(), e.getValue().id()); + if (supportsNodeAssignment()) { + for (Map.Entry e : remoteNodeToInfo().entrySet()) { + createIndexForNode(remoteClient(), e.getKey(), e.getValue().id()); + } + } else { + createIndexForNode(remoteClient(), null, null); } } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 26ba9361eea01..e9826b146baa5 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -34,8 +34,11 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.stream.Collectors; +import static org.elasticsearch.TransportVersions.INITIAL_ELASTICSEARCH_9_0; import static org.elasticsearch.TransportVersions.NEW_SEMANTIC_QUERY_INTERCEPTORS; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; import static org.elasticsearch.test.ListMatcher.matchesList; @@ -90,7 +93,7 @@ protected AllSupportedFieldsTestCase(MappedFieldType.FieldExtractPreference extr this.indexMode = indexMode; } - protected record NodeInfo(String cluster, String id, TransportVersion version) {} + protected record NodeInfo(String cluster, String id, TransportVersion version, Set roles) {} private static Map nodeToInfo; @@ -114,6 +117,17 @@ protected boolean fetchDenseVectorAggMetricDoubleIfFns() throws IOException { return clusterHasCapability("GET", "/_query", List.of(), List.of("DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS")).orElse(false); } + private static Boolean supportsNodeAssignment; + + protected boolean supportsNodeAssignment() throws IOException { + if (supportsNodeAssignment == null) { + supportsNodeAssignment = allNodeToInfo().values() + .stream() + .allMatch(i -> i.roles.contains("index") && i.roles.contains("search")); + } + return supportsNodeAssignment; + } + /** * Map from node name to information about the node. */ @@ -129,10 +143,13 @@ protected static Map fetchNodeToInfo(RestClient client, String for (Map.Entry n : nodes.entrySet()) { String id = (String) n.getKey(); Map nodeInfo = (Map) n.getValue(); - logger.error("DAFADFS {}", nodeInfo); String nodeName = (String) extractValue(nodeInfo, "name"); TransportVersion transportVersion = TransportVersion.fromId((Integer) extractValue(nodeInfo, "transport_version")); - nodeToInfo.put(nodeName, new NodeInfo(cluster, id, transportVersion)); + List roles = (List) nodeInfo.get("roles"); + nodeToInfo.put( + nodeName, + new NodeInfo(cluster, id, transportVersion, roles.stream().map(Object::toString).collect(Collectors.toSet())) + ); } return nodeToInfo; @@ -140,8 +157,12 @@ protected static Map fetchNodeToInfo(RestClient client, String @Before public void createIndices() throws IOException { - for (Map.Entry e : nodeToInfo().entrySet()) { - createIndexForNode(client(), e.getKey(), e.getValue().id()); + if (supportsNodeAssignment()) { + for (Map.Entry e : nodeToInfo().entrySet()) { + createIndexForNode(client(), e.getKey(), e.getValue().id()); + } + } else { + createIndexForNode(client(), null, null); } } @@ -173,8 +194,8 @@ public final void testFetchAll() throws IOException { assertMap(nameToType(columns), expectedColumns); MapMatcher expectedAllValues = matchesMap(); - for (Map.Entry e : allNodeToInfo().entrySet()) { - String nodeName = e.getKey(); + for (Map.Entry e : expectedIndices().entrySet()) { + String indexName = e.getKey(); NodeInfo nodeInfo = e.getValue(); MapMatcher expectedValues = matchesMap(); for (DataType type : DataType.values()) { @@ -185,12 +206,12 @@ public final void testFetchAll() throws IOException { } expectedValues = expectedValues.entry("_id", any(String.class)) .entry("_ignored", nullValue()) - .entry("_index", expectedIndex(nodeName, nodeInfo)) + .entry("_index", indexName) .entry("_index_mode", indexMode.toString()) .entry("_score", 0.0) .entry("_source", matchesMap().extraOk()) .entry("_version", 1); - expectedAllValues = expectedAllValues.entry(expectedIndex(nodeName, nodeInfo), expectedValues); + expectedAllValues = expectedAllValues.entry(indexName, expectedValues); } assertMap(indexToRow(columns, values), expectedAllValues); profileLogger.clearProfile(); @@ -228,7 +249,8 @@ public final void testFetchDenseVector() throws IOException { EntityUtils.toString(e.getResponse().getEntity()), anyOf( containsString("Unknown function [v_l2_norm]"), - containsString("Cannot use field [f_dense_vector] with unsupported type") + containsString("Cannot use field [f_dense_vector] with unsupported type"), + containsString("doesn't understand data type [DENSE_VECTOR]") ) ); // Failure is expected and fine @@ -241,13 +263,13 @@ public final void testFetchDenseVector() throws IOException { assertMap(nameToType(columns), expectedColumns); MapMatcher expectedAllValues = matchesMap(); - for (Map.Entry e : allNodeToInfo().entrySet()) { - String nodeName = e.getKey(); + for (Map.Entry e : expectedIndices().entrySet()) { + String indexName = e.getKey(); NodeInfo nodeInfo = e.getValue(); MapMatcher expectedValues = matchesMap(); expectedValues = expectedValues.entry("f_dense_vector", expectedDenseVector(nodeInfo.version)); - expectedValues = expectedValues.entry("_index", expectedIndex(nodeName, nodeInfo)); - expectedAllValues = expectedAllValues.entry(expectedIndex(nodeName, nodeInfo), expectedValues); + expectedValues = expectedValues.entry("_index", indexName); + expectedAllValues = expectedAllValues.entry(indexName, expectedValues); } assertMap(indexToRow(columns, values), expectedAllValues); } @@ -275,7 +297,10 @@ private Map esql(String query) throws IOException { } protected void createIndexForNode(RestClient client, String nodeName, String nodeId) throws IOException { - String indexName = indexMode + "_" + nodeName.toLowerCase(Locale.ROOT); + String indexName = indexMode.toString(); + if (nodeName != null) { + indexName += "_" + nodeName.toLowerCase(Locale.ROOT); + } if (false == indexExists(client, indexName)) { createAllTypesIndex(client, indexName, nodeId); createAllTypesDoc(client, indexName); @@ -291,7 +316,9 @@ private void createAllTypesIndex(RestClient client, String indexName, String nod if (indexMode == IndexMode.TIME_SERIES) { config.field("routing_path", "f_keyword"); } - config.field("routing.allocation.include._id", nodeId); + if (nodeId != null) { + config.field("routing.allocation.include._id", nodeId); + } config.endObject(); config.endObject(); } @@ -409,7 +436,7 @@ private Matcher expectedValue(TransportVersion version, DataType type) throws } private Matcher> expectedDenseVector(TransportVersion version) { - return version.onOrAfter(NEW_SEMANTIC_QUERY_INTERCEPTORS) + return version.onOrAfter(INITIAL_ELASTICSEARCH_9_0) ? matchesList().item(0.5).item(10.0).item(5.9999995) : matchesList().item(0.04283529).item(0.85670584).item(0.5140235); } @@ -506,11 +533,29 @@ private boolean syntheticSourceByDefault() { }; } - private String expectedIndex(String nodeName, NodeInfo nodeInfo) { - String expectedIndex = indexMode + "_" + nodeName.toLowerCase(Locale.ROOT); - if (nodeInfo.cluster == null) { - return expectedIndex; + private Map expectedIndices() throws IOException { + logger.error("ADFADF NOCOMMIT"); + Map result = new TreeMap<>(); + if (supportsNodeAssignment()) { + logger.error("supports {}", allNodeToInfo()); + for (Map.Entry e : allNodeToInfo().entrySet()) { + String name = indexMode + "_" + e.getKey(); + if (e.getValue().cluster != null) { + name = e.getValue().cluster + ":" + name; + } + result.put(name, e.getValue()); + } + } else { + logger.error("one per {}", allNodeToInfo()); + for (Map.Entry e : allNodeToInfo().entrySet()) { + String name = indexMode.toString(); + if (e.getValue().cluster != null) { + name = e.getValue().cluster + ":" + name; + } + // We should only end up with one per cluster + result.put(name, new NodeInfo(e.getValue().cluster, null, e.getValue().version(), null)); + } } - return nodeInfo.cluster + ":" + expectedIndex; + return result; } } From d2a50643778994937af5d7c9fab3571980b33b38 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 25 Sep 2025 21:28:52 +0000 Subject: [PATCH 34/38] [CI] Auto commit changes from spotless --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index e9826b146baa5..4be6d8f69b92b 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -39,7 +39,6 @@ import java.util.stream.Collectors; import static org.elasticsearch.TransportVersions.INITIAL_ELASTICSEARCH_9_0; -import static org.elasticsearch.TransportVersions.NEW_SEMANTIC_QUERY_INTERCEPTORS; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; From d172d269c465aba90b488312ce5fa8ae41193898 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 25 Sep 2025 17:29:13 -0400 Subject: [PATCH 35/38] Test case --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index e9826b146baa5..4be6d8f69b92b 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -39,7 +39,6 @@ import java.util.stream.Collectors; import static org.elasticsearch.TransportVersions.INITIAL_ELASTICSEARCH_9_0; -import static org.elasticsearch.TransportVersions.NEW_SEMANTIC_QUERY_INTERCEPTORS; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; From 280daf81f26c17a8683a1cb3667f260dc768631c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 26 Sep 2025 08:36:09 -0400 Subject: [PATCH 36/38] updates one last time please please --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 4be6d8f69b92b..8d8ccf3aa74e1 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -38,6 +38,7 @@ import java.util.TreeMap; import java.util.stream.Collectors; +import static org.elasticsearch.TransportVersions.INDEX_SOURCE; import static org.elasticsearch.TransportVersions.INITIAL_ELASTICSEARCH_9_0; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; import static org.elasticsearch.test.ListMatcher.matchesList; @@ -120,9 +121,12 @@ protected boolean fetchDenseVectorAggMetricDoubleIfFns() throws IOException { protected boolean supportsNodeAssignment() throws IOException { if (supportsNodeAssignment == null) { + for (NodeInfo i : allNodeToInfo().values()) { + logger.error("NOCOMMIT {}", i); + } supportsNodeAssignment = allNodeToInfo().values() .stream() - .allMatch(i -> i.roles.contains("index") && i.roles.contains("search")); + .allMatch(i -> (i.roles.contains("index") && i.roles.contains("search")) || (i.roles.contains("data"))); } return supportsNodeAssignment; } @@ -435,7 +439,7 @@ private Matcher expectedValue(TransportVersion version, DataType type) throws } private Matcher> expectedDenseVector(TransportVersion version) { - return version.onOrAfter(INITIAL_ELASTICSEARCH_9_0) + return version.onOrAfter(INDEX_SOURCE) // *after* 9.1 ? matchesList().item(0.5).item(10.0).item(5.9999995) : matchesList().item(0.04283529).item(0.85670584).item(0.5140235); } From 85771e7c6cc4dc84d2983e0208ba4f9f4ce7af2a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 26 Sep 2025 12:43:09 +0000 Subject: [PATCH 37/38] [CI] Auto commit changes from spotless --- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 8d8ccf3aa74e1..f4bbdb99f00ab 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -39,7 +39,6 @@ import java.util.stream.Collectors; import static org.elasticsearch.TransportVersions.INDEX_SOURCE; -import static org.elasticsearch.TransportVersions.INITIAL_ELASTICSEARCH_9_0; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; From d694176a06ac405b0f11a7b150a4672b2be3a89c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 26 Sep 2025 09:05:35 -0400 Subject: [PATCH 38/38] spotless --- .../org/elasticsearch/xpack/esql/action/EsqlCapabilities.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 5213c691e13fd..f8e7e68442811 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1554,7 +1554,7 @@ public enum Cap { INLINE_STATS_FIX_OPTIMIZED_AS_LOCAL_RELATION(INLINESTATS_V11.enabled), DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS - + ; private final boolean enabled;