From f1f6d2b6e29d78587e1ba05e2dce1f91ef79c421 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 31 Oct 2025 10:50:11 +0100 Subject: [PATCH 1/4] Add createdVersion for types --- ..._exponential_histogram_created_version.csv | 1 + .../resources/transport/upper_bounds/9.3.csv | 2 +- .../xpack/esql/core/type/DataType.java | 91 ++++++++++++++----- .../esql/core/type/SupportedVersion.java | 56 +++++++----- 4 files changed, 105 insertions(+), 45 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv diff --git a/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv b/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv new file mode 100644 index 0000000000000..44b76d9df26ec --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv @@ -0,0 +1 @@ +9206000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 1acd7ceede226..11f8a9ef6bff8 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -text_similarity_rank_docs_explain_chunks,9205000 +esql_exponential_histogram_created_version,9206000 diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 8270605b97ca6..7eb0f3335c1c8 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -46,6 +46,10 @@ * {@link DataType#OBJECT}) * *

Process for adding a new data type

+ * We assume that the data type is already supported in ES indices, but not in + * ES|QL. Types that aren't yet enabled in ES will require some adjustments to + * the process. + *

* Note: it is not expected that all the following steps be done in a single PR. * Use capabilities to gate tests as you go, and use as many PRs as you think * appropriate. New data types are complex, and smaller PRs will make reviews @@ -53,13 +57,14 @@ *

- * There are some additional steps that should be taken when removing the - * feature flag and getting ready for a release: + * There are some additional steps that should be taken when getting ready for a release: * *

- * Snapshot builds treat these as always supported so that we can write tests before actually - * turning on the support for the type. Mixed/multi cluster tests with older nodes have to be - * skipped based on capabilites, as always. + * Snapshot builds treat these as supported starting from the version they were created on, + * so that we can write tests before actually turning on the support for the type. + * Mixed/multi cluster tests with older nodes should be skipped based on SNAPSHOT-only + * capabilites, as always. */ // We used to have a feature-flag based override, so that in-development types could be // turned on for testing in release builds. If needed, it's fine to bring this back, but we // need to make sure that other checks for types being under construction are also overridden. - // Check usage of this constant to be sure. - SupportedVersion UNDER_CONSTRUCTION = new SupportedVersion() { - @Override - public boolean supportedOn(TransportVersion version, boolean currentBuildIsSnapshot) { - return currentBuildIsSnapshot; - } + // Check usage of this method to be sure. + static SupportedVersion underConstruction(TransportVersion createdVersion) { + return new SupportedVersion() { + @Override + public boolean supportedOn(TransportVersion version, boolean currentBuildIsSnapshot) { + return currentBuildIsSnapshot && version.supports(createdVersion); + } - @Override - public String toString() { - return "UnderConstruction"; - } - }; + @Override + public String toString() { + return "UnderConstruction"; + } + + @Override + public boolean underConstruction() { + return true; + } + }; + } /** * Types that are supported starting with the given version. *

- * Snapshot builds treat these as always supported, so that any existing tests using them - * continue to work. Otherwise, we'd have to update bwc tests to skip older versions based - * on a capability check, which can be error-prone and risks turning off an unrelated bwc test. + * Snapshot builds treat these as supported from their created version onward, so that any existing tests + * using them should continue to work. */ - static SupportedVersion supportedSince(TransportVersion supportedVersion) { + static SupportedVersion supportedSince(TransportVersion createdVersion, TransportVersion supportedVersion) { + assert supportedVersion.onOrAfter(createdVersion) : "support for a type cannot be enabled before its initial creation"; return new SupportedVersion() { @Override public boolean supportedOn(TransportVersion version, boolean currentBuildIsSnapshot) { - return version.supports(supportedVersion) || currentBuildIsSnapshot; + return currentBuildIsSnapshot ? version.supports(createdVersion) : version.supports(supportedVersion); } @Override From 0b146c8c8ac13ce8244af36fc33c3b4f4c76f9b9 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 31 Oct 2025 10:24:29 +0000 Subject: [PATCH 2/4] [CI] Update transport version definitions --- .../referable/esql_exponential_histogram_created_version.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv b/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv index 44b76d9df26ec..44625f112155b 100644 --- a/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv +++ b/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv @@ -1 +1 @@ -9206000 +9206000,9185006 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 23dfcd8d57f3a..e538a2223eaba 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -esql_resolve_fields_response_used,9185005 +esql_exponential_histogram_created_version,9185006 From ec2b572223596e181b8b4f91bfa4166e81aa4a85 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 2 Dec 2025 15:05:49 +0000 Subject: [PATCH 3/4] [CI] Update transport version definitions --- .../referable/esql_exponential_histogram_created_version.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv b/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv index b42c7bdc031c3..affd2fcf679eb 100644 --- a/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv +++ b/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv @@ -1 +1 @@ -9232000 +9232000,9185012 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 6f76f928f4bb0..70975c93723a0 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -esql_use_minimum_version_for_enrich_resolution,9185011 +esql_exponential_histogram_created_version,9185012 From 70f0868e814ef08efbc9a22a9e528e981e6f8500 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 2 Dec 2025 18:39:07 +0100 Subject: [PATCH 4/4] Do not create new transport version for exp histo --- .../esql_exponential_histogram_created_version.csv | 1 - server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.3.csv | 2 +- .../org/elasticsearch/xpack/esql/core/type/DataType.java | 9 ++++++--- 4 files changed, 8 insertions(+), 6 deletions(-) delete mode 100644 server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv diff --git a/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv b/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv deleted file mode 100644 index affd2fcf679eb..0000000000000 --- a/server/src/main/resources/transport/definitions/referable/esql_exponential_histogram_created_version.csv +++ /dev/null @@ -1 +0,0 @@ -9232000,9185012 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 70975c93723a0..6f76f928f4bb0 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -esql_exponential_histogram_created_version,9185012 +esql_use_minimum_version_for_enrich_resolution,9185011 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 24801cab0b10b..106690c0c4247 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -esql_exponential_histogram_created_version,9232000 +esql_use_minimum_version_for_enrich_resolution,9231000 diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 44f8e317f0ab8..b9ba250658f5a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -378,7 +378,7 @@ public enum DataType implements Writeable { builder().esType("exponential_histogram") .estimatedSize(16 * 160)// guess 160 buckets (OTEL default for positive values only histograms) with 16 bytes per bucket .docValues() - .underConstruction(DataTypesTransportVersions.ESQL_EXPONENTIAL_HISTOGRAM_CREATED_VERSION) + .underConstruction(DataTypesTransportVersions.RESOLVE_FIELDS_RESPONSE_USED_TV) ), /** @@ -1041,8 +1041,11 @@ public static class DataTypesTransportVersions { "esql_aggregate_metric_double_created_version" ); - public static final TransportVersion ESQL_EXPONENTIAL_HISTOGRAM_CREATED_VERSION = TransportVersion.fromName( - "esql_exponential_histogram_created_version" + /** + * First transport version after the PR that introduced the exponential histogram data type. + */ + public static final TransportVersion RESOLVE_FIELDS_RESPONSE_USED_TV = TransportVersion.fromName( + "esql_resolve_fields_response_used" ); } }