From e57027468da6ca99865194b59e75e7a656f3708e Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 5 Mar 2025 17:27:24 -0500 Subject: [PATCH 01/34] wip - update release notes automation to use new markdown format --- .../release/GenerateReleaseNotesTask.java | 5 +- .../release/ReleaseNotesGenerator.java | 22 +- .../internal/release/ReleaseToolsPlugin.java | 17 +- .../resources/templates/breaking-changes.md | 30 + .../main/resources/templates/release-notes.md | 47 + docs/release-notes/breaking-changes.md | 93 +- docs/release-notes/index.md | 816 +++++++++++++----- 7 files changed, 770 insertions(+), 260 deletions(-) create mode 100644 build-tools-internal/src/main/resources/templates/breaking-changes.md create mode 100644 build-tools-internal/src/main/resources/templates/release-notes.md diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index 87aa03f0445c5..668f5730b3957 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -139,10 +139,11 @@ public void executeTask() throws IOException { ); LOGGER.info("Generating breaking changes / deprecations notes..."); - BreakingChangesGenerator.update( + ReleaseNotesGenerator.update( this.breakingChangesTemplate.get().getAsFile(), this.breakingChangesMigrationFile.get().getAsFile(), - entries + qualifiedVersion, + changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) ); LOGGER.info("Updating migration/index..."); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java index df0e35c487f54..4f615fc9167e8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java @@ -39,9 +39,11 @@ public class ReleaseNotesGenerator { TYPE_LABELS.put("breaking", "Breaking changes"); TYPE_LABELS.put("breaking-java", "Breaking Java changes"); TYPE_LABELS.put("bug", "Bug fixes"); + TYPE_LABELS.put("fixes", "Fixes"); TYPE_LABELS.put("deprecation", "Deprecations"); TYPE_LABELS.put("enhancement", "Enhancements"); TYPE_LABELS.put("feature", "New features"); + TYPE_LABELS.put("features-enhancements", "Features and enhancements"); TYPE_LABELS.put("new-aggregation", "New aggregation"); TYPE_LABELS.put("regression", "Regressions"); TYPE_LABELS.put("upgrade", "Upgrades"); @@ -63,16 +65,34 @@ static String generateFile(String template, QualifiedVersion version, Set>> buildChangelogBreakdown(Set changelogs) { Map>> changelogsByTypeByArea = changelogs.stream() .collect( groupingBy( // Entries with breaking info are always put in the breaking section - entry -> entry.getBreaking() == null ? entry.getType() : "breaking", + entry -> getTypeFromEntry(entry), TreeMap::new, // Group changelogs for each type by their team area groupingBy( diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index 22af46a4e5ccd..7bfc8f70470f8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -80,26 +80,17 @@ public void apply(Project project) { task.setReleaseNotesIndexTemplate(projectDirectory.file(RESOURCES + "templates/release-notes-index.asciidoc")); task.setReleaseNotesIndexFile(projectDirectory.file("docs/reference/release-notes.asciidoc")); - task.setReleaseNotesTemplate(projectDirectory.file(RESOURCES + "templates/release-notes.asciidoc")); + task.setReleaseNotesTemplate(projectDirectory.file(RESOURCES + "templates/release-notes.md")); task.setReleaseNotesFile( - projectDirectory.file( - String.format( - "docs/reference/release-notes/%d.%d.%d.asciidoc", - version.getMajor(), - version.getMinor(), - version.getRevision() - ) - ) + projectDirectory.file("docs/release-notes/index.md") ); task.setReleaseHighlightsTemplate(projectDirectory.file(RESOURCES + "templates/release-highlights.asciidoc")); task.setReleaseHighlightsFile(projectDirectory.file("docs/reference/release-notes/highlights.asciidoc")); - task.setBreakingChangesTemplate(projectDirectory.file(RESOURCES + "templates/breaking-changes.asciidoc")); + task.setBreakingChangesTemplate(projectDirectory.file(RESOURCES + "templates/breaking-changes.md")); task.setBreakingChangesMigrationFile( - projectDirectory.file( - String.format("docs/reference/migration/migrate_%d_%d.asciidoc", version.getMajor(), version.getMinor()) - ) + projectDirectory.file("docs/release-notes/breaking-changes.md") ); task.setMigrationIndexTemplate(projectDirectory.file(RESOURCES + "templates/migration-index.asciidoc")); task.setMigrationIndexFile(projectDirectory.file("docs/reference/migration/index.asciidoc")); diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.md b/build-tools-internal/src/main/resources/templates/breaking-changes.md new file mode 100644 index 0000000000000..066a6b5ff27ac --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.md @@ -0,0 +1,30 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes.html +--- + +# Elasticsearch breaking changes [elasticsearch-breaking-changes] +Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. + +To learn how to upgrade, check out . + +% ## Next version [elasticsearch-nextversion-breaking-changes] +% **Release date:** Month day, year + +## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-breaking-changes] +**Release date:** April 01, 2025 +<% for (team in changelogsByTypeByArea['breaking'].keySet()) { + print "\n${team}:\n"; + + for (change in changelogsByTypeByArea['breaking'][team]) { + print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" + if (change.issues != null && change.issues.empty == false) { + print change.issues.size() == 1 ? " (issue: " : " (issues: " + print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") + print ")" + } + print "\n" + } +} +print "\n\n" diff --git a/build-tools-internal/src/main/resources/templates/release-notes.md b/build-tools-internal/src/main/resources/templates/release-notes.md new file mode 100644 index 0000000000000..1ca6cc76b4252 --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/release-notes.md @@ -0,0 +1,47 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-${unqualifiedVersion}.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-${version.major}.${version.minor}.html +--- + +# Elasticsearch release notes [elasticsearch-release-notes] + +Review the changes, fixes, and more in each version of Elasticsearch. + +To check for security updates, go to [Security announcements for the Elastic stack](https://discuss.elastic.co/c/announcements/security-announcements/31). + +% Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. + +% ## version.next [felasticsearch-next-release-notes] +% **Release date:** Month day, year + +% ### Features and enhancements [elasticsearch-next-features-enhancements] +% * + +% ### Fixes [elasticsearch-next-fixes] +% * + +## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-release-notes] +**Release date:** April 01, 2025 +<% +for (changeType in changelogsByTypeByArea.keySet()) { +%> +### ${ TYPE_LABELS.getOrDefault(changeType, 'No mapping for TYPE_LABELS[' + changeType + ']') } [elasticsearch-${versionWithoutSeparator}-${changeType}] +<% for (team in changelogsByTypeByArea[changeType].keySet()) { + print "\n${team}:\n"; + + for (change in changelogsByTypeByArea[changeType][team]) { + print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" + if (change.issues != null && change.issues.empty == false) { + print change.issues.size() == 1 ? " (issue: " : " (issues: " + print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") + print ")" + } + print "\n" + } +} +} +print "\n\n" diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index db628a5271595..d3754dab5e43b 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -5,58 +5,99 @@ mapped_pages: --- # Elasticsearch breaking changes [elasticsearch-breaking-changes] -Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. +Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. To learn how to upgrade, check out . % ## Next version [elasticsearch-nextversion-breaking-changes] % **Release date:** Month day, year -## 9.0.0 [elasticsearch-900-breaking-changes] -**Release date:** March 25, 2025 +## 9.1.0 [elasticsearch-910-breaking-changes] +**Release date:** April 01, 2025 -Allocation -: * Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831) +Aggregations: +* Remove date histogram boolean support [#118484](https://github.com/elastic/elasticsearch/pull/118484) + +Allocation: +* Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831) * Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting [#114207](https://github.com/elastic/elasticsearch/pull/114207) -* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: [#88978](https://github.com/elastic/elasticsearch/issues/88978)) +* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: {es-issue}88978[#88978]) -Analysis -: * Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146) -* The *german2* stemmer is now an alias for the *german* snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614) -* The *persian* analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: [#113050](https://github.com/elastic/elasticsearch/issues/113050)) +Analysis: +* Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146) +* The 'german2' stemmer is now an alias for the 'german' snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614) +* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: {es-issue}113050[#113050]) * The Korean dictionary for Nori has been updated [#114124](https://github.com/elastic/elasticsearch/pull/114124) +Authentication: +* Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password +will prevent node from starting [#118366](https://github.com/elastic/elasticsearch/pull/118366) -Cluster Coordination -: * Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903) +Cluster Coordination: +* Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903) +EQL: +* Set allow_partial_search_results=true by default [#120267](https://github.com/elastic/elasticsearch/pull/120267) -Highlighting -: * Remove support for deprecated `force_source` highlighting parameter [#116943](https://github.com/elastic/elasticsearch/pull/116943) +Extract&Transform: +* Restrict Connector APIs to manage/monitor_connector privileges [#119863](https://github.com/elastic/elasticsearch/pull/119863) +Highlighting: +* Remove support for deprecated `force_source` highlighting parameter [#116943](https://github.com/elastic/elasticsearch/pull/116943) -Indices APIs -: * Apply more strict parsing of actions in bulk API [#115923](https://github.com/elastic/elasticsearch/pull/115923) +Indices APIs: +* Apply more strict parsing of actions in bulk API [#115923](https://github.com/elastic/elasticsearch/pull/115923) * Remove deprecated local attribute from alias APIs [#115393](https://github.com/elastic/elasticsearch/pull/115393) +* Remove the ability to read frozen indices [#120108](https://github.com/elastic/elasticsearch/pull/120108) +* Remove unfreeze REST endpoint [#119227](https://github.com/elastic/elasticsearch/pull/119227) + +Infra/Core: +* Change Elasticsearch timeouts to 429 response instead of 5xx [#116026](https://github.com/elastic/elasticsearch/pull/116026) +* Limit `ByteSizeUnit` to 2 decimals [#120142](https://github.com/elastic/elasticsearch/pull/120142) +* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: {es-issue}104574[#104574]) +* Remove any references to org.elasticsearch.core.RestApiVersion#V_7 [#118103](https://github.com/elastic/elasticsearch/pull/118103) +Infra/Logging: +* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: {es-issue}83251[#83251]) -Infra/REST API -: * Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: [#89387](https://github.com/elastic/elasticsearch/issues/89387)) +Infra/Metrics: +* Deprecated tracing.apm.* settings got removed. [#119926](https://github.com/elastic/elasticsearch/pull/119926) +Infra/REST API: +* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: {es-issue}89387[#89387]) -Ingest Node -: * Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077) +Ingest Node: +* Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077) * Remove ignored fallback option on GeoIP processor [#116112](https://github.com/elastic/elasticsearch/pull/116112) +Logs: +* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: {es-issue}106489[#106489]) + +Machine Learning: +* Disable machine learning on macOS x86_64 [#104125](https://github.com/elastic/elasticsearch/pull/104125) + +Mapping: +* Remove support for type, fields, `copy_to` and boost in metadata field definition [#118825](https://github.com/elastic/elasticsearch/pull/118825) +* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: {es-issue}118596[#118596]) + +Search: +* Adjust `random_score` default field to `_seq_no` field [#118671](https://github.com/elastic/elasticsearch/pull/118671) +* Change Semantic Text To Act Like A Normal Text Field [#120813](https://github.com/elastic/elasticsearch/pull/120813) +* Remove legacy params from range query [#116970](https://github.com/elastic/elasticsearch/pull/116970) + +Snapshot/Restore: +* Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting [#114202](https://github.com/elastic/elasticsearch/pull/114202) -Mapping -: * Remove support for type, fields, `copy_to` and boost in metadata field definition [#116944](https://github.com/elastic/elasticsearch/pull/116944) +TLS: +* Remove TLSv1.1 from default protocols [#121731](https://github.com/elastic/elasticsearch/pull/121731) +Transform: +* Remove `data_frame_transforms` roles [#117519](https://github.com/elastic/elasticsearch/pull/117519) -Search -: * Remove legacy params from range query [#116970](https://github.com/elastic/elasticsearch/pull/116970) +Vector Search: +* Remove old `_knn_search` tech preview API in v9 [#118104](https://github.com/elastic/elasticsearch/pull/118104) +Watcher: +* Removing support for types field in watcher search [#120748](https://github.com/elastic/elasticsearch/pull/120748) -Snapshot/Restore -: * Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting [#114202](https://github.com/elastic/elasticsearch/pull/114202) diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 3d7a4e9473f3f..7f530eb4c80b5 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -3,8 +3,8 @@ navigation_title: "Elasticsearch" mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-9.0.0.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.0.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-9.1.0.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.1.html --- # Elasticsearch release notes [elasticsearch-release-notes] @@ -24,277 +24,657 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * -## 9.0.0 [elasticsearch-900-release-notes] -**Release date:** March 25, 2025 +## 9.1.0 [elasticsearch-910-release-notes] +**Release date:** April 01, 2025 -### Features and enhancements [elasticsearch-900-features-enhancements] -Allocation -: * Only publish desired balance gauges on master [#115383](https://github.com/elastic/elasticsearch/pull/115383) +### Breaking changes [elasticsearch-910-breaking] +Aggregations: +* Remove date histogram boolean support [#118484](https://github.com/elastic/elasticsearch/pull/118484) -Authorization -: * Add a `monitor_stats` privilege and allow that privilege for remote cluster privileges [#114964](https://github.com/elastic/elasticsearch/pull/114964) -* [Security Solution] Add `create_index` to `kibana_system` role for index/DS `.logs-endpoint.action.responses-*` [#115241](https://github.com/elastic/elasticsearch/pull/115241) - - -CRUD -: * Suppress merge-on-recovery for older indices [#113462](https://github.com/elastic/elasticsearch/pull/113462) - +Allocation: +* Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831) +* Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting [#114207](https://github.com/elastic/elasticsearch/pull/114207) +* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: {es-issue}88978[#88978]) -Data streams -: * Adding a deprecation info API warning for data streams with old indices [#116447](https://github.com/elastic/elasticsearch/pull/116447) -* Apm-data: disable date_detection for all apm data streams [#116995](https://github.com/elastic/elasticsearch/pull/116995) -* Add default ILM policies and switch to ILM for apm-data plugin [#115687](https://github.com/elastic/elasticsearch/pull/115687) +Analysis: +* Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146) +* The 'german2' stemmer is now an alias for the 'german' snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614) +* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: {es-issue}113050[#113050]) +* The Korean dictionary for Nori has been updated [#114124](https://github.com/elastic/elasticsearch/pull/114124) +Authentication: +* Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password +will prevent node from starting [#118366](https://github.com/elastic/elasticsearch/pull/118366) -Distributed -: * Metrics for incremental bulk splits [#116765](https://github.com/elastic/elasticsearch/pull/116765) -* Use Azure blob batch API to delete blobs in batches [#114566](https://github.com/elastic/elasticsearch/pull/114566) +Cluster Coordination: +* Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903) +EQL: +* Set allow_partial_search_results=true by default [#120267](https://github.com/elastic/elasticsearch/pull/120267) -ES|QL -: * Add ES|QL `bit_length` function [#115792](https://github.com/elastic/elasticsearch/pull/115792) -* ESQL: Honor skip_unavailable setting for nonmatching indices errors at planning time [#116348](https://github.com/elastic/elasticsearch/pull/116348) (issue: [#114531](https://github.com/elastic/elasticsearch/issues/114531)) -* ESQL: Remove parent from `FieldAttribute` [#112881](https://github.com/elastic/elasticsearch/pull/112881) -* ESQL: extract common filter from aggs [#115678](https://github.com/elastic/elasticsearch/pull/115678) -* ESQL: optimise aggregations filtered by false/null into evals [#115858](https://github.com/elastic/elasticsearch/pull/115858) -* ES|QL CCS uses `skip_unavailable` setting for handling disconnected remote clusters [#115266](https://github.com/elastic/elasticsearch/pull/115266) (issue: [#114531](https://github.com/elastic/elasticsearch/issues/114531)) -* ES|QL: add metrics for functions [#114620](https://github.com/elastic/elasticsearch/pull/114620) -* Esql Enable Date Nanos (tech preview) [#117080](https://github.com/elastic/elasticsearch/pull/117080) -* Support partial sort fields in TopN pushdown [#116043](https://github.com/elastic/elasticsearch/pull/116043) (issue: [#114515](https://github.com/elastic/elasticsearch/issues/114515)) -* [ES|QL] Implicit casting string literal to intervals [#115814](https://github.com/elastic/elasticsearch/pull/115814) (issue: [#115352](https://github.com/elastic/elasticsearch/issues/115352)) -* Add support for `BYTE_LENGTH` scalar function [#116591](https://github.com/elastic/elasticsearch/pull/116591) -* Esql/lookup join grammar [#116515](https://github.com/elastic/elasticsearch/pull/116515) -* Remove snapshot build restriction for match and qstr functions [#114482](https://github.com/elastic/elasticsearch/pull/114482) +Extract&Transform: +* Restrict Connector APIs to manage/monitor_connector privileges [#119863](https://github.com/elastic/elasticsearch/pull/119863) +Highlighting: +* Remove support for deprecated `force_source` highlighting parameter [#116943](https://github.com/elastic/elasticsearch/pull/116943) -Health -: * Increase `replica_unassigned_buffer_time` default from 3s to 5s [#112834](https://github.com/elastic/elasticsearch/pull/112834) +Indices APIs: +* Apply more strict parsing of actions in bulk API [#115923](https://github.com/elastic/elasticsearch/pull/115923) +* Remove deprecated local attribute from alias APIs [#115393](https://github.com/elastic/elasticsearch/pull/115393) +* Remove the ability to read frozen indices [#120108](https://github.com/elastic/elasticsearch/pull/120108) +* Remove unfreeze REST endpoint [#119227](https://github.com/elastic/elasticsearch/pull/119227) +Infra/Core: +* Change Elasticsearch timeouts to 429 response instead of 5xx [#116026](https://github.com/elastic/elasticsearch/pull/116026) +* Limit `ByteSizeUnit` to 2 decimals [#120142](https://github.com/elastic/elasticsearch/pull/120142) +* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: {es-issue}104574[#104574]) +* Remove any references to org.elasticsearch.core.RestApiVersion#V_7 [#118103](https://github.com/elastic/elasticsearch/pull/118103) -Indices APIs -: * Ensure class resource stream is closed in `ResourceUtils` [#116437](https://github.com/elastic/elasticsearch/pull/116437) +Infra/Logging: +* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: {es-issue}83251[#83251]) +Infra/Metrics: +* Deprecated tracing.apm.* settings got removed. [#119926](https://github.com/elastic/elasticsearch/pull/119926) -Inference -: * Add version prefix to Inference Service API path [#117095](https://github.com/elastic/elasticsearch/pull/117095) +Infra/REST API: +* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: {es-issue}89387[#89387]) +Ingest Node: +* Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077) +* Remove ignored fallback option on GeoIP processor [#116112](https://github.com/elastic/elasticsearch/pull/116112) -Infra/Circuit Breakers -: * Add link to Circuit Breaker "Data too large" exception message [#113561](https://github.com/elastic/elasticsearch/pull/113561) +Logs: +* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: {es-issue}106489[#106489]) +Machine Learning: +* Disable machine learning on macOS x86_64 [#104125](https://github.com/elastic/elasticsearch/pull/104125) -Infra/Core -: * Support for unsigned 64 bit numbers in Cpu stats [#114681](https://github.com/elastic/elasticsearch/pull/114681) (issue: [#112274](https://github.com/elastic/elasticsearch/issues/112274)) +Mapping: +* Remove support for type, fields, `copy_to` and boost in metadata field definition [#118825](https://github.com/elastic/elasticsearch/pull/118825) +* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: {es-issue}118596[#118596]) +Search: +* Adjust `random_score` default field to `_seq_no` field [#118671](https://github.com/elastic/elasticsearch/pull/118671) +* Change Semantic Text To Act Like A Normal Text Field [#120813](https://github.com/elastic/elasticsearch/pull/120813) +* Remove legacy params from range query [#116970](https://github.com/elastic/elasticsearch/pull/116970) -Infra/Metrics -: * Add `ensureGreen` test method for use with `adminClient` [#113425](https://github.com/elastic/elasticsearch/pull/113425) +Snapshot/Restore: +* Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting [#114202](https://github.com/elastic/elasticsearch/pull/114202) +TLS: +* Remove TLSv1.1 from default protocols [#121731](https://github.com/elastic/elasticsearch/pull/121731) -Infra/Scripting -: * Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts [#114002](https://github.com/elastic/elasticsearch/pull/114002) +Transform: +* Remove `data_frame_transforms` roles [#117519](https://github.com/elastic/elasticsearch/pull/117519) +Vector Search: +* Remove old `_knn_search` tech preview API in v9 [#118104](https://github.com/elastic/elasticsearch/pull/118104) -Ingest Node -: * Add postal_code support to the City and Enterprise databases [#114193](https://github.com/elastic/elasticsearch/pull/114193) -* Add support for registered country fields for maxmind geoip databases [#114521](https://github.com/elastic/elasticsearch/pull/114521) -* Adding support for additional mapping to simulate ingest API [#114742](https://github.com/elastic/elasticsearch/pull/114742) -* Adding support for simulate ingest mapping adddition for indices with mappings that do not come from templates [#115359](https://github.com/elastic/elasticsearch/pull/115359) -* Support IPinfo database configurations [#114548](https://github.com/elastic/elasticsearch/pull/114548) -* Support more maxmind fields in the geoip processor [#114268](https://github.com/elastic/elasticsearch/pull/114268) +Watcher: +* Removing support for types field in watcher search [#120748](https://github.com/elastic/elasticsearch/pull/120748) +### Deprecations [elasticsearch-910-deprecation] -Logs -: * Add logsdb telemetry [#115994](https://github.com/elastic/elasticsearch/pull/115994) -* Add num docs and size to logsdb telemetry [#116128](https://github.com/elastic/elasticsearch/pull/116128) -* Feature: re-structure document ID generation favoring _id inverted index compression [#104683](https://github.com/elastic/elasticsearch/pull/104683) +ES|QL: +* Drop support for brackets from METADATA syntax [#119846](https://github.com/elastic/elasticsearch/pull/119846) (issue: {es-issue}115401[#115401]) +Ingest Node: +* Fix `_type` deprecation on simulate pipeline API [#116259](https://github.com/elastic/elasticsearch/pull/116259) -Machine Learning -: * Add DeBERTa-V2/V3 tokenizer [#111852](https://github.com/elastic/elasticsearch/pull/111852) -* Add special case for elastic reranker in inference API [#116962](https://github.com/elastic/elasticsearch/pull/116962) -* Adding inference endpoint validation for `AzureAiStudioService` [#113713](https://github.com/elastic/elasticsearch/pull/113713) -* Adds support for `input_type` field to Vertex inference service [#116431](https://github.com/elastic/elasticsearch/pull/116431) -* Enable built-in Inference Endpoints and default for Semantic Text [#116931](https://github.com/elastic/elasticsearch/pull/116931) -* Increase default `queue_capacity` to 10_000 and decrease max `queue_capacity` to 100_000 [#115041](https://github.com/elastic/elasticsearch/pull/115041) -* Inference duration and error metrics [#115876](https://github.com/elastic/elasticsearch/pull/115876) -* Remove all mentions of eis and gateway and deprecate flags that do [#116692](https://github.com/elastic/elasticsearch/pull/116692) -* [Inference API] Add API to get configuration of inference services [#114862](https://github.com/elastic/elasticsearch/pull/114862) -* [Inference API] Improve chunked results error message [#115807](https://github.com/elastic/elasticsearch/pull/115807) +Machine Learning: +* Add deprecation warning for flush API [#121667](https://github.com/elastic/elasticsearch/pull/121667) (issue: {es-issue}121506[#121506]) +* Removing index alias creation for deprecated transforms notification index [#117583](https://github.com/elastic/elasticsearch/pull/117583) +* [Inference API] Deprecate elser service [#113216](https://github.com/elastic/elasticsearch/pull/113216) +Rollup: +* Emit deprecation warning when executing one of the rollup APIs [#113131](https://github.com/elastic/elasticsearch/pull/113131) -Network -: * Allow http unsafe buffers by default [#116115](https://github.com/elastic/elasticsearch/pull/116115) +Search: +* Deprecate Behavioral Analytics CRUD apis [#122960](https://github.com/elastic/elasticsearch/pull/122960) +Security: +* Deprecate certificate based remote cluster security model [#120806](https://github.com/elastic/elasticsearch/pull/120806) -Recovery -: * Attempt to clean up index before remote transfer [#115142](https://github.com/elastic/elasticsearch/pull/115142) (issue: [#104473](https://github.com/elastic/elasticsearch/issues/104473)) -* Trigger merges after recovery [#113102](https://github.com/elastic/elasticsearch/pull/113102) +### Features and enhancements [elasticsearch-910-features-enhancements] +Allocation: +* Add a not-master state for desired balance [#116904](https://github.com/elastic/elasticsearch/pull/116904) +* Introduce `AllocationBalancingRoundSummaryService` [#120957](https://github.com/elastic/elasticsearch/pull/120957) +* Only publish desired balance gauges on master [#115383](https://github.com/elastic/elasticsearch/pull/115383) +* Reset relocation/allocation failure counter on node join/shutdown [#119968](https://github.com/elastic/elasticsearch/pull/119968) -Reindex -: * Change Reindexing metrics unit from millis to seconds [#115721](https://github.com/elastic/elasticsearch/pull/115721) +Authentication: +* Allow `SSHA-256` for API key credential hash [#120997](https://github.com/elastic/elasticsearch/pull/120997) +Authorization: +* Allow kibana_system user to manage .reindexed-v8-internal.alerts indices [#118959](https://github.com/elastic/elasticsearch/pull/118959) +* Do not fetch reserved roles from native store when Get Role API is called [#121971](https://github.com/elastic/elasticsearch/pull/121971) +* Grant necessary Kibana application privileges to `reporting_user` role [#118058](https://github.com/elastic/elasticsearch/pull/118058) +* Make reserved built-in roles queryable [#117581](https://github.com/elastic/elasticsearch/pull/117581) +* [Security Solution] Add `create_index` to `kibana_system` role for index/DS `.logs-endpoint.action.responses-*` [#115241](https://github.com/elastic/elasticsearch/pull/115241) +* [Security Solution] allows `kibana_system` user to manage .reindexed-v8-* Security Solution indices [#119054](https://github.com/elastic/elasticsearch/pull/119054) + +CCS: +* Resolve/cluster allows querying for cluster info only (no index expression required) [#119898](https://github.com/elastic/elasticsearch/pull/119898) + +CRUD: +* Metrics for indexing failures due to version conflicts [#119067](https://github.com/elastic/elasticsearch/pull/119067) +* Suppress merge-on-recovery for older indices [#113462](https://github.com/elastic/elasticsearch/pull/113462) + +Cluster Coordination: +* Include `clusterApplyListener` in long cluster apply warnings [#120087](https://github.com/elastic/elasticsearch/pull/120087) + +Data streams: +* Add action to create index from a source index [#118890](https://github.com/elastic/elasticsearch/pull/118890) +* Add index and reindex request settings to speed up reindex [#119780](https://github.com/elastic/elasticsearch/pull/119780) +* Add rest endpoint for `create_from_source_index` [#119250](https://github.com/elastic/elasticsearch/pull/119250) +* Add sanity check to `ReindexDatastreamIndexAction` [#120231](https://github.com/elastic/elasticsearch/pull/120231) +* Adding a migration reindex cancel API [#118291](https://github.com/elastic/elasticsearch/pull/118291) +* Adding get migration reindex status [#118267](https://github.com/elastic/elasticsearch/pull/118267) +* Consistent mapping for OTel log and event bodies [#120547](https://github.com/elastic/elasticsearch/pull/120547) +* Filter deprecated settings when making dest index [#120163](https://github.com/elastic/elasticsearch/pull/120163) +* Ignore closed indices for reindex [#120244](https://github.com/elastic/elasticsearch/pull/120244) +* Improve how reindex data stream index action handles api blocks [#120084](https://github.com/elastic/elasticsearch/pull/120084) +* Initial work on `ReindexDatastreamIndexAction` [#116996](https://github.com/elastic/elasticsearch/pull/116996) +* Make `requests_per_second` configurable to throttle reindexing [#120207](https://github.com/elastic/elasticsearch/pull/120207) +* Optimized index sorting for OTel logs [#119504](https://github.com/elastic/elasticsearch/pull/119504) +* Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As A Warning [#120629](https://github.com/elastic/elasticsearch/pull/120629) +* Update data stream deprecations warnings to new format and filter searchable snapshots from response [#118562](https://github.com/elastic/elasticsearch/pull/118562) + +Distributed: +* Make various alias retrieval APIs wait for cluster to unblock [#117230](https://github.com/elastic/elasticsearch/pull/117230) +* Metrics for incremental bulk splits [#116765](https://github.com/elastic/elasticsearch/pull/116765) +* Use Azure blob batch API to delete blobs in batches [#114566](https://github.com/elastic/elasticsearch/pull/114566) -Relevance -: * Add query rules retriever [#114855](https://github.com/elastic/elasticsearch/pull/114855) -* Add tracking for query rule types [#116357](https://github.com/elastic/elasticsearch/pull/116357) - +EQL: +* Add support for partial shard results [#116388](https://github.com/elastic/elasticsearch/pull/116388) +* Optional named arguments for function in map [#118619](https://github.com/elastic/elasticsearch/pull/118619) + +ES|QL: +* Add ES|QL cross-cluster query telemetry collection [#119474](https://github.com/elastic/elasticsearch/pull/119474) +* Add a `LicenseAware` interface for licensed Nodes [#118931](https://github.com/elastic/elasticsearch/pull/118931) (issue: {es-issue}117405[#117405]) +* Add a `PostAnalysisAware,` distribute verification [#119798](https://github.com/elastic/elasticsearch/pull/119798) +* Add a standard deviation aggregating function: STD_DEV [#116531](https://github.com/elastic/elasticsearch/pull/116531) +* Add cluster level reduction [#117731](https://github.com/elastic/elasticsearch/pull/117731) +* Add initial grammar and changes for FORK [#121948](https://github.com/elastic/elasticsearch/pull/121948) +* Add nulls support to Categorize [#117655](https://github.com/elastic/elasticsearch/pull/117655) +* Allow partial results in ES|QL [#121942](https://github.com/elastic/elasticsearch/pull/121942) +* Allow skip shards with `_tier` and `_index` in ES|QL [#123728](https://github.com/elastic/elasticsearch/pull/123728) +* Async search responses have CCS metadata while searches are running [#117265](https://github.com/elastic/elasticsearch/pull/117265) +* Check for early termination in Driver [#118188](https://github.com/elastic/elasticsearch/pull/118188) +* Do not serialize `EsIndex` in plan [#119580](https://github.com/elastic/elasticsearch/pull/119580) +* ESQL - Add Match function options [#120360](https://github.com/elastic/elasticsearch/pull/120360) +* ESQL - Allow full text functions disjunctions for non-full text functions [#120291](https://github.com/elastic/elasticsearch/pull/120291) +* ESQL - Remove restrictions for disjunctions in full text functions [#118544](https://github.com/elastic/elasticsearch/pull/118544) +* ESQL - enabling scoring with METADATA `_score` [#113120](https://github.com/elastic/elasticsearch/pull/113120) +* ESQL Add esql hash function [#117989](https://github.com/elastic/elasticsearch/pull/117989) +* ESQL Support IN operator for Date nanos [#119772](https://github.com/elastic/elasticsearch/pull/119772) (issue: {es-issue}118578[#118578]) +* ESQL: CATEGORIZE as a `BlockHash` [#114317](https://github.com/elastic/elasticsearch/pull/114317) +* ESQL: Enable async get to support formatting [#111104](https://github.com/elastic/elasticsearch/pull/111104) (issue: {es-issue}110926[#110926]) +* ESQL: Enterprise license enforcement for CCS [#118102](https://github.com/elastic/elasticsearch/pull/118102) +* ES|QL: Partial result on demand for async queries [#118122](https://github.com/elastic/elasticsearch/pull/118122) +* Enable KQL function as a tech preview [#119730](https://github.com/elastic/elasticsearch/pull/119730) +* Enable LOOKUP JOIN in non-snapshot builds [#121193](https://github.com/elastic/elasticsearch/pull/121193) (issue: {es-issue}121185[#121185]) +* Enable node-level reduction by default [#119621](https://github.com/elastic/elasticsearch/pull/119621) +* Enable physical plan verification [#118114](https://github.com/elastic/elasticsearch/pull/118114) +* Ensure cluster string could be quoted [#120355](https://github.com/elastic/elasticsearch/pull/120355) +* Esql - Support date nanos in date extract function [#120727](https://github.com/elastic/elasticsearch/pull/120727) (issue: {es-issue}110000[#110000]) +* Esql - support date nanos in date format function [#120143](https://github.com/elastic/elasticsearch/pull/120143) (issue: {es-issue}109994[#109994]) +* Esql Support date nanos on date diff function [#120645](https://github.com/elastic/elasticsearch/pull/120645) (issue: {es-issue}109999[#109999]) +* Esql bucket function for date nanos [#118474](https://github.com/elastic/elasticsearch/pull/118474) (issue: {es-issue}118031[#118031]) +* Esql compare nanos and millis [#118027](https://github.com/elastic/elasticsearch/pull/118027) (issue: {es-issue}116281[#116281]) +* Esql implicit casting for date nanos [#118697](https://github.com/elastic/elasticsearch/pull/118697) (issue: {es-issue}118476[#118476]) +* Expand type compatibility for match function and operator [#117555](https://github.com/elastic/elasticsearch/pull/117555) +* Extend `TranslationAware` to all pushable expressions [#120192](https://github.com/elastic/elasticsearch/pull/120192) +* Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: {es-issue}122967[#122967]) +* Hash functions [#118938](https://github.com/elastic/elasticsearch/pull/118938) +* Implement a `MetricsAware` interface [#121074](https://github.com/elastic/elasticsearch/pull/121074) +* Implement runtime skip_unavailable=true [#121240](https://github.com/elastic/elasticsearch/pull/121240) +* Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) +* Introduce a pre-mapping logical plan processing step [#121260](https://github.com/elastic/elasticsearch/pull/121260) +* LOOKUP JOIN using field-caps for field mapping [#117246](https://github.com/elastic/elasticsearch/pull/117246) +* Lookup join on multiple join fields not yet supported [#118858](https://github.com/elastic/elasticsearch/pull/118858) +* Move scoring in ES|QL out of snapshot [#120354](https://github.com/elastic/elasticsearch/pull/120354) +* Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` [#119889](https://github.com/elastic/elasticsearch/pull/119889) +* Push down filter passed lookup join [#118410](https://github.com/elastic/elasticsearch/pull/118410) +* Render `aggregate_metric_double` [#122660](https://github.com/elastic/elasticsearch/pull/122660) +* Resume Driver on cancelled or early finished [#120020](https://github.com/elastic/elasticsearch/pull/120020) +* Retry ES|QL node requests on shard level failures [#120774](https://github.com/elastic/elasticsearch/pull/120774) +* Rewrite TO_UPPER/TO_LOWER comparisons [#118870](https://github.com/elastic/elasticsearch/pull/118870) (issue: {es-issue}118304[#118304]) +* ST_EXTENT aggregation [#117451](https://github.com/elastic/elasticsearch/pull/117451) (issue: {es-issue}104659[#104659]) +* ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape [#118802](https://github.com/elastic/elasticsearch/pull/118802) +* Smarter field caps with subscribable listener [#116755](https://github.com/elastic/elasticsearch/pull/116755) +* Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions [#116964](https://github.com/elastic/elasticsearch/pull/116964) (issue: {es-issue}104875[#104875]) +* Support partial results in CCS in ES|QL [#122708](https://github.com/elastic/elasticsearch/pull/122708) +* Support partial sort fields in TopN pushdown [#116043](https://github.com/elastic/elasticsearch/pull/116043) (issue: {es-issue}114515[#114515]) +* Support some stats on aggregate_metric_double [#120343](https://github.com/elastic/elasticsearch/pull/120343) (issue: {es-issue}110649[#110649]) +* Support subset of metrics in aggregate metric double [#121805](https://github.com/elastic/elasticsearch/pull/121805) +* Take named parameters for identifier and pattern out of snapshot [#121850](https://github.com/elastic/elasticsearch/pull/121850) +* Term query for ES|QL [#117359](https://github.com/elastic/elasticsearch/pull/117359) +* Update grammar to rely on `indexPattern` instead of identifier in join target [#120494](https://github.com/elastic/elasticsearch/pull/120494) +* `_score` should not be a reserved attribute in ES|QL [#118435](https://github.com/elastic/elasticsearch/pull/118435) (issue: {es-issue}118460[#118460]) + +Engine: +* Defer unpromotable shard refreshes until index refresh blocks are cleared [#120642](https://github.com/elastic/elasticsearch/pull/120642) +* POC mark read-only [#119743](https://github.com/elastic/elasticsearch/pull/119743) + +Experiences: +* Integrate IBM watsonx to Inference API for re-ranking task [#117176](https://github.com/elastic/elasticsearch/pull/117176) + +Extract&Transform: +* [Connector API] Support hard deletes with new URL param in delete endpoint [#120200](https://github.com/elastic/elasticsearch/pull/120200) +* [Connector API] Support soft-deletes of connectors [#118669](https://github.com/elastic/elasticsearch/pull/118669) +* [Connector APIs] Enforce index prefix for managed connectors [#117778](https://github.com/elastic/elasticsearch/pull/117778) + +Geo: +* Optimize indexing points with index and doc values set to true [#120271](https://github.com/elastic/elasticsearch/pull/120271) + +Health: +* Add health indicator impact to `HealthPeriodicLogger` [#122390](https://github.com/elastic/elasticsearch/pull/122390) +* Increase `replica_unassigned_buffer_time` default from 3s to 5s [#112834](https://github.com/elastic/elasticsearch/pull/112834) + +Highlighting: +* Add Highlighter for Semantic Text Fields [#118064](https://github.com/elastic/elasticsearch/pull/118064) + +ILM+SLM: +* Add a `replicate_for` option to the ILM `searchable_snapshot` action [#119003](https://github.com/elastic/elasticsearch/pull/119003) +* Improve SLM Health Indicator to cover missing snapshot [#121370](https://github.com/elastic/elasticsearch/pull/121370) + +Indices APIs: +* Add `remove_index_block` arg to `_create_from` api [#120548](https://github.com/elastic/elasticsearch/pull/120548) +* Remove index blocks by default in `create_from` [#120643](https://github.com/elastic/elasticsearch/pull/120643) +* Run `TransportGetComponentTemplateAction` on local node [#116868](https://github.com/elastic/elasticsearch/pull/116868) +* Run `TransportGetComposableIndexTemplate` on local node [#119830](https://github.com/elastic/elasticsearch/pull/119830) +* Run `TransportGetIndexTemplateAction` on local node [#119837](https://github.com/elastic/elasticsearch/pull/119837) +* introduce new categories for deprecated resources in deprecation API [#120505](https://github.com/elastic/elasticsearch/pull/120505) + +Inference: +* Add version prefix to Inference Service API path [#117095](https://github.com/elastic/elasticsearch/pull/117095) +* Remove Elastic Inference Service feature flag and deprecated setting [#120842](https://github.com/elastic/elasticsearch/pull/120842) +* Update sparse text embeddings API route for Inference Service [#118025](https://github.com/elastic/elasticsearch/pull/118025) +* [Elastic Inference Service] Add ElasticInferenceService Unified ChatCompletions Integration [#118871](https://github.com/elastic/elasticsearch/pull/118871) +* [Inference API] Rename `model_id` prop to model in EIS sparse inference request body [#122272](https://github.com/elastic/elasticsearch/pull/122272) + +Infra/CLI: +* Strengthen encryption for elasticsearch-keystore tool to AES 256 [#119749](https://github.com/elastic/elasticsearch/pull/119749) + +Infra/Circuit Breakers: +* Add link to Circuit Breaker "Data too large" exception message [#113561](https://github.com/elastic/elasticsearch/pull/113561) + +Infra/Core: +* Add support for specifying reindexing script for system index migration [#119001](https://github.com/elastic/elasticsearch/pull/119001) +* Change default Docker image to be based on UBI minimal instead of Ubuntu [#116739](https://github.com/elastic/elasticsearch/pull/116739) +* Improve size limiting string message [#122427](https://github.com/elastic/elasticsearch/pull/122427) +* Infrastructure for assuming cluster features in the next major version [#118143](https://github.com/elastic/elasticsearch/pull/118143) + +Infra/Metrics: +* Add `ensureGreen` test method for use with `adminClient` [#113425](https://github.com/elastic/elasticsearch/pull/113425) + +Infra/REST API: +* A new query parameter `?include_source_on_error` was added for create / index, update and bulk REST APIs to control +if to include the document source in the error response in case of parsing errors. The default value is `true`. [#120725](https://github.com/elastic/elasticsearch/pull/120725) + +Infra/Scripting: +* Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts [#114002](https://github.com/elastic/elasticsearch/pull/114002) + +Infra/Settings: +* Introduce `IndexSettingDeprecatedInV8AndRemovedInV9` Setting property [#120334](https://github.com/elastic/elasticsearch/pull/120334) +* Run `TransportClusterGetSettingsAction` on local node [#119831](https://github.com/elastic/elasticsearch/pull/119831) + +Ingest Node: +* Allow setting the `type` in the reroute processor [#122409](https://github.com/elastic/elasticsearch/pull/122409) (issue: {es-issue}121553[#121553]) +* Optimize `IngestCtxMap` construction [#120833](https://github.com/elastic/elasticsearch/pull/120833) +* Optimize `IngestDocMetadata` `isAvailable` [#120753](https://github.com/elastic/elasticsearch/pull/120753) +* Optimize `IngestDocument` `FieldPath` allocation [#120573](https://github.com/elastic/elasticsearch/pull/120573) +* Optimize some per-document hot paths in the geoip processor [#120824](https://github.com/elastic/elasticsearch/pull/120824) +* Returning ignored fields in the simulate ingest API [#117214](https://github.com/elastic/elasticsearch/pull/117214) +* Run `GetPipelineTransportAction` on local node [#120445](https://github.com/elastic/elasticsearch/pull/120445) +* Run `TransportEnrichStatsAction` on local node [#121256](https://github.com/elastic/elasticsearch/pull/121256) +* Run `TransportGetEnrichPolicyAction` on local node [#121124](https://github.com/elastic/elasticsearch/pull/121124) +* Run template simulation actions on local node [#120038](https://github.com/elastic/elasticsearch/pull/120038) + +License: +* Bump `TrialLicenseVersion` to allow starting new trial on 9.0 [#120198](https://github.com/elastic/elasticsearch/pull/120198) + +Logs: +* Add LogsDB option to route on sort fields [#116687](https://github.com/elastic/elasticsearch/pull/116687) +* Add a new index setting to skip recovery source when synthetic source is enabled [#114618](https://github.com/elastic/elasticsearch/pull/114618) +* Configure index sorting through index settings for logsdb [#118968](https://github.com/elastic/elasticsearch/pull/118968) (issue: {es-issue}118686[#118686]) +* Optimize loading mappings when determining synthetic source usage and whether host.name can be sorted on. [#120055](https://github.com/elastic/elasticsearch/pull/120055) + +Machine Learning: +* Add DeBERTa-V2/V3 tokenizer [#111852](https://github.com/elastic/elasticsearch/pull/111852) +* Add Inference Unified API for chat completions for OpenAI [#117589](https://github.com/elastic/elasticsearch/pull/117589) +* Add Jina AI API to do inference for Embedding and Rerank models [#118652](https://github.com/elastic/elasticsearch/pull/118652) +* Add enterprise license check for Inference API actions [#119893](https://github.com/elastic/elasticsearch/pull/119893) +* Adding chunking settings to `IbmWatsonxService` [#114914](https://github.com/elastic/elasticsearch/pull/114914) +* Adding default endpoint for Elastic Rerank [#117939](https://github.com/elastic/elasticsearch/pull/117939) +* Adding elser default endpoint for EIS [#122066](https://github.com/elastic/elasticsearch/pull/122066) +* Adding endpoint creation validation for all task types to remaining services [#115020](https://github.com/elastic/elasticsearch/pull/115020) +* Adding endpoint creation validation to `ElasticInferenceService` [#117642](https://github.com/elastic/elasticsearch/pull/117642) +* Adding integration for VoyageAI embeddings and rerank models [#122134](https://github.com/elastic/elasticsearch/pull/122134) +* Adding support for binary embedding type to Cohere service embedding type [#120751](https://github.com/elastic/elasticsearch/pull/120751) +* Adding support for specifying embedding type to Jina AI service settings [#121548](https://github.com/elastic/elasticsearch/pull/121548) +* Check for presence of error object when validating streaming responses from integrations in the inference API [#118375](https://github.com/elastic/elasticsearch/pull/118375) +* ES|QL `change_point` processing command [#120998](https://github.com/elastic/elasticsearch/pull/120998) +* ES|QL categorize with multiple groupings [#118173](https://github.com/elastic/elasticsearch/pull/118173) +* Ignore failures from renormalizing buckets in read-only index [#118674](https://github.com/elastic/elasticsearch/pull/118674) +* Inference duration and error metrics [#115876](https://github.com/elastic/elasticsearch/pull/115876) +* Migrate stream to core error parsing [#120722](https://github.com/elastic/elasticsearch/pull/120722) +* Remove all mentions of eis and gateway and deprecate flags that do [#116692](https://github.com/elastic/elasticsearch/pull/116692) +* Remove deprecated sort from reindex operation within dataframe analytics procedure [#117606](https://github.com/elastic/elasticsearch/pull/117606) +* Retry on `ClusterBlockException` on transform destination index [#118194](https://github.com/elastic/elasticsearch/pull/118194) +* Support mTLS for the Elastic Inference Service integration inside the inference API [#119679](https://github.com/elastic/elasticsearch/pull/119679) +* [Inference API] Add node-local rate limiting for the inference API [#120400](https://github.com/elastic/elasticsearch/pull/120400) +* [Inference API] fix spell words: covertToString to convertToString [#119922](https://github.com/elastic/elasticsearch/pull/119922) + +Mapping: +* Add Optional Source Filtering to Source Loaders [#113827](https://github.com/elastic/elasticsearch/pull/113827) +* Add option to store `sparse_vector` outside `_source` [#117917](https://github.com/elastic/elasticsearch/pull/117917) +* Enable the use of nested field type with index.mode=time_series [#122224](https://github.com/elastic/elasticsearch/pull/122224) (issue: {es-issue}120874[#120874]) +* Improved error message when index field type is invalid [#122860](https://github.com/elastic/elasticsearch/pull/122860) +* Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields [#119546](https://github.com/elastic/elasticsearch/pull/119546) +* Store arrays offsets for ip fields natively with synthetic source [#122999](https://github.com/elastic/elasticsearch/pull/122999) +* Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. [#113757](https://github.com/elastic/elasticsearch/pull/113757) +* Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields [#122637](https://github.com/elastic/elasticsearch/pull/122637) +* Use `FallbackSyntheticSourceBlockLoader` for number fields [#122280](https://github.com/elastic/elasticsearch/pull/122280) + +Network: +* Allow http unsafe buffers by default [#116115](https://github.com/elastic/elasticsearch/pull/116115) +* Http stream activity tracker and exceptions handling [#119564](https://github.com/elastic/elasticsearch/pull/119564) +* Remove HTTP content copies [#117303](https://github.com/elastic/elasticsearch/pull/117303) +* `ConnectTransportException` returns retryable BAD_GATEWAY [#118681](https://github.com/elastic/elasticsearch/pull/118681) (issue: {es-issue}118320[#118320]) + +Ranking: +* Add a generic `rescorer` retriever based on the search request's rescore functionality [#118585](https://github.com/elastic/elasticsearch/pull/118585) (issue: {es-issue}118327[#118327]) +* Set default reranker for text similarity reranker to Elastic reranker [#120551](https://github.com/elastic/elasticsearch/pull/120551) + +Recovery: +* Allow archive and searchable snapshots indices in N-2 version [#118941](https://github.com/elastic/elasticsearch/pull/118941) +* Trigger merges after recovery [#113102](https://github.com/elastic/elasticsearch/pull/113102) -Search -: * Add Search Phase APM metrics [#113194](https://github.com/elastic/elasticsearch/pull/113194) -* Add `docvalue_fields` Support for `dense_vector` Fields [#114484](https://github.com/elastic/elasticsearch/pull/114484) (issue: [#108470](https://github.com/elastic/elasticsearch/issues/108470)) -* Add initial support for `semantic_text` field type [#113920](https://github.com/elastic/elasticsearch/pull/113920) -* Adds access to flags no_sub_matches and no_overlapping_matches to hyphenation-decompounder-tokenfilter [#115459](https://github.com/elastic/elasticsearch/pull/115459) (issue: [#97849](https://github.com/elastic/elasticsearch/issues/97849)) -* Better sizing `BytesRef` for Strings in Queries [#115655](https://github.com/elastic/elasticsearch/pull/115655) -* Enable `_tier` based coordinator rewrites for all indices (not just mounted indices) [#115797](https://github.com/elastic/elasticsearch/pull/115797) +Reindex: +* Change Reindexing metrics unit from millis to seconds [#115721](https://github.com/elastic/elasticsearch/pull/115721) + +Relevance: +* Add Multi-Field Support for Semantic Text Fields [#120128](https://github.com/elastic/elasticsearch/pull/120128) + +Search: +* Account for the `SearchHit` source in circuit breaker [#121920](https://github.com/elastic/elasticsearch/pull/121920) (issue: {es-issue}89656[#89656]) +* Add match support for `semantic_text` fields [#117839](https://github.com/elastic/elasticsearch/pull/117839) +* Add support for `sparse_vector` queries against `semantic_text` fields [#118617](https://github.com/elastic/elasticsearch/pull/118617) +* Add support for knn vector queries on `semantic_text` fields [#119011](https://github.com/elastic/elasticsearch/pull/119011) +* Adding linear retriever to support weighted sums of sub-retrievers [#120222](https://github.com/elastic/elasticsearch/pull/120222) +* Address and remove any references of RestApiVersion version 7 [#117572](https://github.com/elastic/elasticsearch/pull/117572) +* Feat: add a user-configurable timeout parameter to the `_resolve/cluster` API [#120542](https://github.com/elastic/elasticsearch/pull/120542) +* Make semantic text part of the text family [#119792](https://github.com/elastic/elasticsearch/pull/119792) * Only aggregations require at least one shard request [#115314](https://github.com/elastic/elasticsearch/pull/115314) -* ESQL - Add match operator (:) [#116819](https://github.com/elastic/elasticsearch/pull/116819) -* Upgrade to Lucene 10 [#114741](https://github.com/elastic/elasticsearch/pull/114741) +* Optionally allow text similarity reranking to fail [#121784](https://github.com/elastic/elasticsearch/pull/121784) +* Prevent data nodes from sending stack traces to coordinator when `error_trace=false` [#118266](https://github.com/elastic/elasticsearch/pull/118266) +* Propagate status codes from shard failures appropriately [#118016](https://github.com/elastic/elasticsearch/pull/118016) (issue: {es-issue}118482[#118482]) +Security: +* Add refresh `.security` index call between security migrations [#114879](https://github.com/elastic/elasticsearch/pull/114879) -Security -: * Add refresh `.security` index call between security migrations [#114879](https://github.com/elastic/elasticsearch/pull/114879) - - -Snapshot/Restore -: * Improve message about insecure S3 settings [#116915](https://github.com/elastic/elasticsearch/pull/116915) +Snapshot/Restore: +* Add IMDSv2 support to `repository-s3` [#117748](https://github.com/elastic/elasticsearch/pull/117748) (issue: {es-issue}105135[#105135]) +* Expose operation and request counts separately in repository stats [#117530](https://github.com/elastic/elasticsearch/pull/117530) (issue: {es-issue}104443[#104443]) * Retry `S3BlobContainer#getRegister` on all exceptions [#114813](https://github.com/elastic/elasticsearch/pull/114813) -* Split searchable snapshot into multiple repo operations [#116918](https://github.com/elastic/elasticsearch/pull/116918) +* Retry internally when CAS upload is throttled [GCS] [#120250](https://github.com/elastic/elasticsearch/pull/120250) (issue: {es-issue}116546[#116546]) * Track shard snapshot progress during node shutdown [#112567](https://github.com/elastic/elasticsearch/pull/112567) - -Vector Search -: * Add support for bitwise inner-product in painless [#116082](https://github.com/elastic/elasticsearch/pull/116082) - -### Fixes [elasticsearch-900-fixes] -Aggregations -: * Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations [#116174](https://github.com/elastic/elasticsearch/pull/116174) (issue: [#115777](https://github.com/elastic/elasticsearch/issues/115777)) - - -Analysis -: * Adjust analyze limit exception to be a `bad_request` [#116325](https://github.com/elastic/elasticsearch/pull/116325) - - -CCS -: * Fix long metric deserialize & add - auto-resize needs to be set manually [#117105](https://github.com/elastic/elasticsearch/pull/117105) (issue: [#116914](https://github.com/elastic/elasticsearch/issues/116914)) - - -CRUD -: * Preserve thread context when waiting for segment generation in RTG [#114623](https://github.com/elastic/elasticsearch/pull/114623) -* Standardize error code when bulk body is invalid [#114869](https://github.com/elastic/elasticsearch/pull/114869) - - -Data streams -: * Add missing header in `put_data_lifecycle` rest-api-spec [#116292](https://github.com/elastic/elasticsearch/pull/116292) - - -EQL -: * Don’t use a `BytesStreamOutput` to copy keys in `BytesRefBlockHash` [#114819](https://github.com/elastic/elasticsearch/pull/114819) (issue: [#114599](https://github.com/elastic/elasticsearch/issues/114599)) - - -ES|QL -: * Added stricter range type checks and runtime warnings for ENRICH [#115091](https://github.com/elastic/elasticsearch/pull/115091) (issues: [#107357](https://github.com/elastic/elasticsearch/issues/107357), [#116799](https://github.com/elastic/elasticsearch/issues/116799)) -* Don’t return TEXT type for functions that take TEXT [#114334](https://github.com/elastic/elasticsearch/pull/114334) (issues: [#111537](https://github.com/elastic/elasticsearch/issues/111537), [#114333](https://github.com/elastic/elasticsearch/issues/114333)) -* ESQL: Fix sorts containing `_source` [#116980](https://github.com/elastic/elasticsearch/pull/116980) (issue: [#116659](https://github.com/elastic/elasticsearch/issues/116659)) -* ESQL: fix the column position in errors [#117153](https://github.com/elastic/elasticsearch/pull/117153) -* ES|QL: Fix stats by constant expression [#114899](https://github.com/elastic/elasticsearch/pull/114899) -* Fix NPE in `EnrichLookupService` on mixed clusters with <8.14 versions [#116583](https://github.com/elastic/elasticsearch/pull/116583) (issues: [#116529](https://github.com/elastic/elasticsearch/issues/116529), [#116544](https://github.com/elastic/elasticsearch/issues/116544)) -* Fix TDigestState.read CB leaks [#114303](https://github.com/elastic/elasticsearch/pull/114303) (issue: [#114194](https://github.com/elastic/elasticsearch/issues/114194)) -* Fixing remote ENRICH by pushing the Enrich inside `FragmentExec` [#114665](https://github.com/elastic/elasticsearch/pull/114665) (issue: [#105095](https://github.com/elastic/elasticsearch/issues/105095)) -* Use `SearchStats` instead of field.isAggregatable in data node planning [#115744](https://github.com/elastic/elasticsearch/pull/115744) (issue: [#115737](https://github.com/elastic/elasticsearch/issues/115737)) -* [ESQL] Fix Binary Comparisons on Date Nanos [#116346](https://github.com/elastic/elasticsearch/pull/116346) -* [ES|QL] To_DatePeriod and To_TimeDuration return better error messages on `union_type` fields [#114934](https://github.com/elastic/elasticsearch/pull/114934) - - -Infra/CLI -: * Fix NPE on plugin sync [#115640](https://github.com/elastic/elasticsearch/pull/115640) (issue: [#114818](https://github.com/elastic/elasticsearch/issues/114818)) - - -Infra/Metrics -: * Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]` [#114177](https://github.com/elastic/elasticsearch/pull/114177) - - -Infra/REST API -: * Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null. [#113413](https://github.com/elastic/elasticsearch/pull/113413) (issue: [#113413](https://github.com/elastic/elasticsearch/issues/113413)) - - -Infra/Settings -: * Don’t allow secure settings in YML config (109115) [#115779](https://github.com/elastic/elasticsearch/pull/115779) (issue: [#109115](https://github.com/elastic/elasticsearch/issues/109115)) - - -Ingest Node -: * Add warning headers for ingest pipelines containing special characters [#114837](https://github.com/elastic/elasticsearch/pull/114837) (issue: [#104411](https://github.com/elastic/elasticsearch/issues/104411)) -* Reducing error-level stack trace logging for normal events in `GeoIpDownloader` [#114924](https://github.com/elastic/elasticsearch/pull/114924) - - -Logs -: * Always check if index mode is logsdb [#116922](https://github.com/elastic/elasticsearch/pull/116922) -* Prohibit changes to index mode, source, and sort settings during resize [#115812](https://github.com/elastic/elasticsearch/pull/115812) - - -Machine Learning -: * Fix bug in ML autoscaling when some node info is unavailable [#116650](https://github.com/elastic/elasticsearch/pull/116650) -* Fix deberta tokenizer bug caused by bug in normalizer [#117189](https://github.com/elastic/elasticsearch/pull/117189) -* Hides `hugging_face_elser` service from the `GET _inference/_services API` [#116664](https://github.com/elastic/elasticsearch/pull/116664) (issue: [#116644](https://github.com/elastic/elasticsearch/issues/116644)) -* Mitigate IOSession timeouts [#115414](https://github.com/elastic/elasticsearch/pull/115414) (issues: [#114385](https://github.com/elastic/elasticsearch/issues/114385), [#114327](https://github.com/elastic/elasticsearch/issues/114327), [#114105](https://github.com/elastic/elasticsearch/issues/114105), [#114232](https://github.com/elastic/elasticsearch/issues/114232)) -* Propagate scoring function through random sampler [#116957](https://github.com/elastic/elasticsearch/pull/116957) (issue: [#110134](https://github.com/elastic/elasticsearch/issues/110134)) -* Update Deberta tokenizer [#116358](https://github.com/elastic/elasticsearch/pull/116358) -* Wait for up to 2 seconds for yellow status before starting search [#115938](https://github.com/elastic/elasticsearch/pull/115938) (issues: [#107777](https://github.com/elastic/elasticsearch/issues/107777), [#105955](https://github.com/elastic/elasticsearch/issues/105955), [#107815](https://github.com/elastic/elasticsearch/issues/107815), [#112191](https://github.com/elastic/elasticsearch/issues/112191)) - - -Mapping -: * Change synthetic source logic for `constant_keyword` [#117182](https://github.com/elastic/elasticsearch/pull/117182) (issue: [#117083](https://github.com/elastic/elasticsearch/issues/117083)) -* Ignore conflicting fields during dynamic mapping update [#114227](https://github.com/elastic/elasticsearch/pull/114227) (issue: [#114228](https://github.com/elastic/elasticsearch/issues/114228)) - - -Network -: * Use underlying `ByteBuf` `refCount` for `ReleasableBytesReference` [#116211](https://github.com/elastic/elasticsearch/pull/116211) - - -Ranking -: * Propagating nested `inner_hits` to the parent compound retriever [#116408](https://github.com/elastic/elasticsearch/pull/116408) (issue: [#116397](https://github.com/elastic/elasticsearch/issues/116397)) - - -Relevance -: * Fix handling of bulk requests with semantic text fields and delete ops [#116942](https://github.com/elastic/elasticsearch/pull/116942) - - -Search -: * Catch and handle disconnect exceptions in search [#115836](https://github.com/elastic/elasticsearch/pull/115836) -* Fields caps does not honour ignore_unavailable [#116021](https://github.com/elastic/elasticsearch/pull/116021) (issue: [#107767](https://github.com/elastic/elasticsearch/issues/107767)) -* Fix handling of time exceeded exception in fetch phase [#116676](https://github.com/elastic/elasticsearch/pull/116676) -* Fix leak in `DfsQueryPhase` and introduce search disconnect stress test [#116060](https://github.com/elastic/elasticsearch/pull/116060) (issue: [#115056](https://github.com/elastic/elasticsearch/issues/115056)) +Stats: +* Run XPack usage actions on local node [#122933](https://github.com/elastic/elasticsearch/pull/122933) + +Store: +* Abort pending deletion on `IndicesService` close [#123569](https://github.com/elastic/elasticsearch/pull/123569) + +Suggesters: +* Extensible Completion Postings Formats [#111494](https://github.com/elastic/elasticsearch/pull/111494) + +TSDB: +* Increase field limit for OTel metrics to 10 000 [#120591](https://github.com/elastic/elasticsearch/pull/120591) + +Transform: +* Add support for `extended_stats` [#120340](https://github.com/elastic/elasticsearch/pull/120340) +* Auto-migrate `max_page_search_size` [#119348](https://github.com/elastic/elasticsearch/pull/119348) +* Create upgrade mode [#117858](https://github.com/elastic/elasticsearch/pull/117858) +* Wait while index is blocked [#119542](https://github.com/elastic/elasticsearch/pull/119542) +* [Deprecation] Add `transform_ids` to outdated index [#120821](https://github.com/elastic/elasticsearch/pull/120821) + +Vector Search: +* Add new experimental `rank_vectors` mapping for late-interaction second order ranking [#118804](https://github.com/elastic/elasticsearch/pull/118804) +* Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields [#122381](https://github.com/elastic/elasticsearch/pull/122381) (issue: {es-issue}117274[#117274]) +* Even better(er) binary quantization [#117994](https://github.com/elastic/elasticsearch/pull/117994) +* KNN vector rescoring for quantized vectors [#116663](https://github.com/elastic/elasticsearch/pull/116663) +* Mark bbq indices as GA and add rolling upgrade integration tests [#121105](https://github.com/elastic/elasticsearch/pull/121105) +* Speed up bit compared with floats or bytes script operations [#117199](https://github.com/elastic/elasticsearch/pull/117199) + +Watcher: +* Run `TransportGetWatcherSettingsAction` on local node [#122857](https://github.com/elastic/elasticsearch/pull/122857) + +### Fixes [elasticsearch-910-fixes] + +Aggregations: +* Disable concurrency when `top_hits` sorts on anything but `_score` [#123610](https://github.com/elastic/elasticsearch/pull/123610) +* Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations [#116174](https://github.com/elastic/elasticsearch/pull/116174) (issue: {es-issue}115777[#115777]) + +Allocation: +* Deduplicate allocation stats calls [#123246](https://github.com/elastic/elasticsearch/pull/123246) +* `DesiredBalanceReconciler` always returns `AllocationStats` [#122458](https://github.com/elastic/elasticsearch/pull/122458) + +Analysis: +* Analyze API to return 400 for wrong custom analyzer [#121568](https://github.com/elastic/elasticsearch/pull/121568) (issue: {es-issue}121443[#121443]) + +Authentication: +* Improve jwt logging on failed auth [#122247](https://github.com/elastic/elasticsearch/pull/122247) + +CAT APIs: +* Fix cat_component_templates documentation [#120487](https://github.com/elastic/elasticsearch/pull/120487) + +CRUD: +* Preserve thread context when waiting for segment generation in RTG [#114623](https://github.com/elastic/elasticsearch/pull/114623) +* Preserve thread context when waiting for segment generation in RTG [#117148](https://github.com/elastic/elasticsearch/pull/117148) +* Reduce license checks in `LicensedWriteLoadForecaster` [#123346](https://github.com/elastic/elasticsearch/pull/123346) (issue: {es-issue}123247[#123247]) + +Data streams: +* Add `_metric_names_hash` field to OTel metric mappings [#120952](https://github.com/elastic/elasticsearch/pull/120952) +* Avoid updating settings version in `MetadataMigrateToDataStreamService` when settings have not changed [#118704](https://github.com/elastic/elasticsearch/pull/118704) +* Block-writes cannot be added after read-only [#119007](https://github.com/elastic/elasticsearch/pull/119007) (issue: {es-issue}119002[#119002]) +* Ensure removal of index blocks does not leave key with null value [#122246](https://github.com/elastic/elasticsearch/pull/122246) +* Match dot prefix of migrated DS backing index with the source index [#120042](https://github.com/elastic/elasticsearch/pull/120042) +* Refresh source index before reindexing data stream index [#120752](https://github.com/elastic/elasticsearch/pull/120752) (issue: {es-issue}120314[#120314]) +* Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover [#122905](https://github.com/elastic/elasticsearch/pull/122905) +* `ReindexDataStreamIndex` bug in assertion caused by reference equality [#121325](https://github.com/elastic/elasticsearch/pull/121325) + +Downsampling: +* Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` [#121727](https://github.com/elastic/elasticsearch/pull/121727) (issues: {es-issue}119696[#119696], {es-issue}96076[#96076]) + +EQL: +* Fix JOIN command validation (not supported) [#122011](https://github.com/elastic/elasticsearch/pull/122011) + +ES|QL: +* Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: {es-issue}122413[#122413]) +* Allow the data type of `null` in filters [#118324](https://github.com/elastic/elasticsearch/pull/118324) (issue: {es-issue}116351[#116351]) +* Avoid over collecting in Limit or Lucene Operator [#123296](https://github.com/elastic/elasticsearch/pull/123296) +* Correct line and column numbers of missing named parameters [#120852](https://github.com/elastic/elasticsearch/pull/120852) +* Drop null columns in text formats [#117643](https://github.com/elastic/elasticsearch/pull/117643) (issue: {es-issue}116848[#116848]) +* ESQL: Fix inconsistent results in using scaled_float field [#122586](https://github.com/elastic/elasticsearch/pull/122586) (issue: {es-issue}122547[#122547]) +* Fix ENRICH validation for use of wildcards [#121911](https://github.com/elastic/elasticsearch/pull/121911) +* Fix ROUND() with unsigned longs throwing in some edge cases [#119536](https://github.com/elastic/elasticsearch/pull/119536) +* Fix TDigestState.read CB leaks [#114303](https://github.com/elastic/elasticsearch/pull/114303) (issue: {es-issue}114194[#114194]) +* Fix TopN row size estimate [#119476](https://github.com/elastic/elasticsearch/pull/119476) (issue: {es-issue}106956[#106956]) +* Fix `AbstractShapeGeometryFieldMapperTests` [#119265](https://github.com/elastic/elasticsearch/pull/119265) (issue: {es-issue}119201[#119201]) +* Fix a bug in TOP [#121552](https://github.com/elastic/elasticsearch/pull/121552) +* Fix async stop sometimes not properly collecting result [#121843](https://github.com/elastic/elasticsearch/pull/121843) (issue: {es-issue}121249[#121249]) +* Fix attribute set equals [#118823](https://github.com/elastic/elasticsearch/pull/118823) +* Fix double lookup failure on ESQL [#115616](https://github.com/elastic/elasticsearch/pull/115616) (issue: {es-issue}111398[#111398]) +* Fix early termination in `LuceneSourceOperator` [#123197](https://github.com/elastic/elasticsearch/pull/123197) +* Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: {es-issue}123430[#123430]) +* Fix functions emitting warnings with no source [#122821](https://github.com/elastic/elasticsearch/pull/122821) (issue: {es-issue}122588[#122588]) +* Fix listener leak in exchange service [#122417](https://github.com/elastic/elasticsearch/pull/122417) (issue: {es-issue}122271[#122271]) +* Fix queries with document level security on lookup indexes [#120617](https://github.com/elastic/elasticsearch/pull/120617) (issue: {es-issue}120509[#120509]) +* Fix writing for LOOKUP status [#119296](https://github.com/elastic/elasticsearch/pull/119296) (issue: {es-issue}119086[#119086]) +* Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: {es-issue}121890[#121890]) +* Limit memory usage of `fold` [#118602](https://github.com/elastic/elasticsearch/pull/118602) +* Limit size of query [#117898](https://github.com/elastic/elasticsearch/pull/117898) +* Reduce iteration complexity for plan traversal [#123427](https://github.com/elastic/elasticsearch/pull/123427) +* Remove duplicated nested commands [#123085](https://github.com/elastic/elasticsearch/pull/123085) +* Remove redundant sorts from execution plan [#121156](https://github.com/elastic/elasticsearch/pull/121156) +* Revert unwanted ES|QL lexer changes from PR #120354 [#120538](https://github.com/elastic/elasticsearch/pull/120538) +* Revive inlinestats [#122257](https://github.com/elastic/elasticsearch/pull/122257) +* Speed up VALUES for many buckets [#123073](https://github.com/elastic/elasticsearch/pull/123073) + +Engine: +* Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: {es-issue}122974[#122974]) + +Health: +* Do not recommend increasing `max_shards_per_node` [#120458](https://github.com/elastic/elasticsearch/pull/120458) + +Indices APIs: +* Add `?master_timeout` to `POST /_ilm/migrate_to_data_tiers` [#120883](https://github.com/elastic/elasticsearch/pull/120883) +* Fix broken yaml test `30_create_from` [#120662](https://github.com/elastic/elasticsearch/pull/120662) +* Include hidden indices in `DeprecationInfoAction` [#118035](https://github.com/elastic/elasticsearch/pull/118035) (issue: {es-issue}118020[#118020]) +* Updates the deprecation info API to not warn about system indices and data streams [#122951](https://github.com/elastic/elasticsearch/pull/122951) + +Inference: +* [Inference API] Put back legacy EIS URL setting [#121207](https://github.com/elastic/elasticsearch/pull/121207) + +Infra/Core: +* Epoch Millis Rounding Down and Not Up 2 [#118353](https://github.com/elastic/elasticsearch/pull/118353) +* Include data streams when converting an existing resource to a system resource [#121392](https://github.com/elastic/elasticsearch/pull/121392) +* Reduce Data Loss in System Indices Migration [#121327](https://github.com/elastic/elasticsearch/pull/121327) +* System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) +* Wrap jackson exception on malformed json string [#114445](https://github.com/elastic/elasticsearch/pull/114445) (issue: {es-issue}114142[#114142]) + +Infra/Logging: +* Move `SlowLogFieldProvider` instantiation to node construction [#117949](https://github.com/elastic/elasticsearch/pull/117949) + +Infra/Metrics: +* Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]` [#114177](https://github.com/elastic/elasticsearch/pull/114177) + +Infra/Plugins: +* Remove unnecessary entitlement [#120959](https://github.com/elastic/elasticsearch/pull/120959) +* Restrict agent entitlements to the system classloader unnamed module [#120546](https://github.com/elastic/elasticsearch/pull/120546) + +Infra/REST API: +* Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null. [#113413](https://github.com/elastic/elasticsearch/pull/113413) (issue: {es-issue}113413[#113413]) + +Infra/Scripting: +* Register mustache size limit setting [#119291](https://github.com/elastic/elasticsearch/pull/119291) + +Infra/Settings: +* Don't allow secure settings in YML config (109115) [#115779](https://github.com/elastic/elasticsearch/pull/115779) (issue: {es-issue}109115[#109115]) + +Ingest: +* Fix `ArrayIndexOutOfBoundsException` in `ShardBulkInferenceActionFilter` [#122538](https://github.com/elastic/elasticsearch/pull/122538) + +Ingest Node: +* Add warning headers for ingest pipelines containing special characters [#114837](https://github.com/elastic/elasticsearch/pull/114837) (issue: {es-issue}104411[#104411]) +* Canonicalize processor names and types in `IngestStats` [#122610](https://github.com/elastic/elasticsearch/pull/122610) +* Deduplicate `IngestStats` and `IngestStats.Stats` identity records when deserializing [#122496](https://github.com/elastic/elasticsearch/pull/122496) +* Fix geoip databases index access after system feature migration [#121196](https://github.com/elastic/elasticsearch/pull/121196) +* Fix geoip databases index access after system feature migration (again) [#122938](https://github.com/elastic/elasticsearch/pull/122938) +* Fix redact processor arraycopy bug [#122640](https://github.com/elastic/elasticsearch/pull/122640) +* Register `IngestGeoIpMetadata` as a NamedXContent [#123079](https://github.com/elastic/elasticsearch/pull/123079) +* Use ordered maps for `PipelineConfiguration` xcontent deserialization [#123403](https://github.com/elastic/elasticsearch/pull/123403) +* apm-data: Use representative count as event.success_count if available [#119995](https://github.com/elastic/elasticsearch/pull/119995) + +Logs: +* Always check if index mode is logsdb [#116922](https://github.com/elastic/elasticsearch/pull/116922) +* Fix issues that prevents using search only snapshots for indices that use index sorting. This is includes Logsdb and time series indices. [#122199](https://github.com/elastic/elasticsearch/pull/122199) +* Use min node version to guard injecting settings in logs provider [#123005](https://github.com/elastic/elasticsearch/pull/123005) (issue: {es-issue}122950[#122950]) + +Machine Learning: +* Add `ElasticInferenceServiceCompletionServiceSettings` [#123155](https://github.com/elastic/elasticsearch/pull/123155) +* Add enterprise license check to inference action for semantic text fields [#122293](https://github.com/elastic/elasticsearch/pull/122293) +* Change format for Unified Chat [#121396](https://github.com/elastic/elasticsearch/pull/121396) +* Fix get all inference endponts not returning multiple endpoints sharing model deployment [#121821](https://github.com/elastic/elasticsearch/pull/121821) +* Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) +* Fixing bedrock event executor terminated cache issue [#118177](https://github.com/elastic/elasticsearch/pull/118177) (issue: {es-issue}117916[#117916]) +* Fixing bug setting index when parsing Google Vertex AI results [#117287](https://github.com/elastic/elasticsearch/pull/117287) +* Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) +* Updates to allow using Cohere binary embedding response in semantic search queries [#121827](https://github.com/elastic/elasticsearch/pull/121827) +* Updating Inference Update API documentation to have the correct PUT method [#121048](https://github.com/elastic/elasticsearch/pull/121048) +* Wait for up to 2 seconds for yellow status before starting search [#115938](https://github.com/elastic/elasticsearch/pull/115938) (issues: {es-issue}107777[#107777], {es-issue}105955[#105955], {es-issue}107815[#107815], {es-issue}112191[#112191]) +* [Inference API] Fix unique ID message for inference ID matches trained model ID [#119543](https://github.com/elastic/elasticsearch/pull/119543) (issue: {es-issue}111312[#111312]) + +Mapping: +* Enable New Semantic Text Format Only On Newly Created Indices [#121556](https://github.com/elastic/elasticsearch/pull/121556) +* Fix propagation of dynamic mapping parameter when applying `copy_to` [#121109](https://github.com/elastic/elasticsearch/pull/121109) (issue: {es-issue}113049[#113049]) +* Fix realtime get of nested fields with synthetic source [#119575](https://github.com/elastic/elasticsearch/pull/119575) (issue: {es-issue}119553[#119553]) +* Fix synthetic source bug that would mishandle nested `dense_vector` fields [#122425](https://github.com/elastic/elasticsearch/pull/122425) +* Merge field mappers when updating mappings with [subobjects:false] [#120370](https://github.com/elastic/elasticsearch/pull/120370) (issue: {es-issue}120216[#120216]) +* Tweak `copy_to` handling in synthetic `_source` to account for nested objects [#120974](https://github.com/elastic/elasticsearch/pull/120974) (issue: {es-issue}120831[#120831]) +* fix stale data in synthetic source for string stored field [#123105](https://github.com/elastic/elasticsearch/pull/123105) (issue: {es-issue}123110[#123110]) + +Network: +* Remove ChunkedToXContentBuilder [#119310](https://github.com/elastic/elasticsearch/pull/119310) (issue: {es-issue}118647[#118647]) + +Search: +* Catch and handle disconnect exceptions in search [#115836](https://github.com/elastic/elasticsearch/pull/115836) +* Fix handling of auto expand replicas for stateless indices [#122365](https://github.com/elastic/elasticsearch/pull/122365) +* Fix leak in `DfsQueryPhase` and introduce search disconnect stress test [#116060](https://github.com/elastic/elasticsearch/pull/116060) (issue: {es-issue}115056[#115056]) +* Fix/QueryBuilderBWCIT_muted_test [#117831](https://github.com/elastic/elasticsearch/pull/117831) +* Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: {es-issue}122186[#122186]) +* In this pr, a 400 error is returned when _source / _seq_no / _feature / _nested_path / _field_names is requested, rather a 5xx [#117229](https://github.com/elastic/elasticsearch/pull/117229) * Inconsistency in the _analyzer api when the index is not included [#115930](https://github.com/elastic/elasticsearch/pull/115930) -* Semantic text simple partial update [#116478](https://github.com/elastic/elasticsearch/pull/116478) -* Updated Date Range to Follow Documentation When Assuming Missing Values [#112258](https://github.com/elastic/elasticsearch/pull/112258) (issue: [#111484](https://github.com/elastic/elasticsearch/issues/111484)) -* Validate missing shards after the coordinator rewrite [#116382](https://github.com/elastic/elasticsearch/pull/116382) -* _validate does not honour ignore_unavailable [#116656](https://github.com/elastic/elasticsearch/pull/116656) (issue: [#116594](https://github.com/elastic/elasticsearch/issues/116594)) +* Remove duplicate code in ESIntegTestCase [#120799](https://github.com/elastic/elasticsearch/pull/120799) +* SearchStatesIt failures reported by CI [#117618](https://github.com/elastic/elasticsearch/pull/117618) (issues: {es-issue}116617[#116617], {es-issue}116618[#116618]) +* Skip fetching _inference_fields field in legacy semantic_text format [#121720](https://github.com/elastic/elasticsearch/pull/121720) +* Test/107515 restore template with match only text mapper it fail [#120392](https://github.com/elastic/elasticsearch/pull/120392) (issue: {es-issue}107515[#107515]) +* Updated Date Range to Follow Documentation When Assuming Missing Values [#112258](https://github.com/elastic/elasticsearch/pull/112258) (issue: {es-issue}111484[#111484]) +* `CrossClusterIT` `testCancel` failure [#117750](https://github.com/elastic/elasticsearch/pull/117750) (issue: {es-issue}108061[#108061]) +* `SearchServiceTests.testParseSourceValidation` failure [#117963](https://github.com/elastic/elasticsearch/pull/117963) + +Snapshot/Restore: +* Add undeclared Azure settings, modify test to exercise them [#118634](https://github.com/elastic/elasticsearch/pull/118634) +* Fork post-snapshot-delete cleanup off master thread [#122731](https://github.com/elastic/elasticsearch/pull/122731) +* Limit number of suppressed S3 deletion errors [#123630](https://github.com/elastic/elasticsearch/pull/123630) (issue: {es-issue}123354[#123354]) +* Retry throttled snapshot deletions [#113237](https://github.com/elastic/elasticsearch/pull/113237) +* Use the system index descriptor in the snapshot blob cache cleanup task [#120937](https://github.com/elastic/elasticsearch/pull/120937) (issue: {es-issue}120518[#120518]) +Stats: +* Fixing serialization of `ScriptStats` `cache_evictions_history` [#123384](https://github.com/elastic/elasticsearch/pull/123384) -Snapshot/Restore -: * Retry throttled snapshot deletions [#113237](https://github.com/elastic/elasticsearch/pull/113237) +Store: +* Do not capture `ClusterChangedEvent` in `IndicesStore` call to #onClusterStateShardsClosed [#120193](https://github.com/elastic/elasticsearch/pull/120193) +Suggesters: +* Return an empty suggestion when suggest phase times out [#122575](https://github.com/elastic/elasticsearch/pull/122575) (issue: {es-issue}122548[#122548]) +* Support duplicate suggestions in completion field [#121324](https://github.com/elastic/elasticsearch/pull/121324) (issue: {es-issue}82432[#82432]) -Vector Search -: * Update Semantic Query To Handle Zero Size Responses [#116277](https://github.com/elastic/elasticsearch/pull/116277) (issue: [#116083](https://github.com/elastic/elasticsearch/issues/116083)) +Transform: +* If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias [#122074](https://github.com/elastic/elasticsearch/pull/122074) (issue: {es-issue}121913[#121913]) +Vector Search: +* Apply default k for knn query eagerly [#118774](https://github.com/elastic/elasticsearch/pull/118774) +* Fix `bbq_hnsw` merge file cleanup on random IO exceptions [#119691](https://github.com/elastic/elasticsearch/pull/119691) (issue: {es-issue}119392[#119392]) +* Knn vector rescoring to sort score docs [#122653](https://github.com/elastic/elasticsearch/pull/122653) (issue: {es-issue}119711[#119711]) -Watcher -: * Watch Next Run Interval Resets On Shard Move or Node Restart [#115102](https://github.com/elastic/elasticsearch/pull/115102) (issue: [#111433](https://github.com/elastic/elasticsearch/issues/111433)) +Watcher: +* Watcher history index has too many indexed fields - [#117701](https://github.com/elastic/elasticsearch/pull/117701) (issue: {es-issue}71479[#71479]) +### Upgrades [elasticsearch-910-upgrade] +Authentication: +* Bump json-smart and oauth2-oidc-sdk [#122737](https://github.com/elastic/elasticsearch/pull/122737) -## Deprecations [deprecation-9.0.0] +Infra/Core: +* Bump major version for feature migration system indices [#117243](https://github.com/elastic/elasticsearch/pull/117243) +* Update ASM 9.7 -> 9.7.1 to support JDK 24 [#118094](https://github.com/elastic/elasticsearch/pull/118094) -Ingest Node -: * Fix `_type` deprecation on simulate pipeline API [#116259](https://github.com/elastic/elasticsearch/pull/116259) +Machine Learning: +* Automatically rollover legacy .ml-anomalies indices [#120913](https://github.com/elastic/elasticsearch/pull/120913) +* Automatically rollover legacy ml indices [#120405](https://github.com/elastic/elasticsearch/pull/120405) +* Change the auditor to write via an alias [#120064](https://github.com/elastic/elasticsearch/pull/120064) +Search: +* Upgrade to Lucene 10 [#114741](https://github.com/elastic/elasticsearch/pull/114741) +* Upgrade to Lucene 10.1.0 [#119308](https://github.com/elastic/elasticsearch/pull/119308) -Machine Learning -: * [Inference API] Deprecate elser service [#113216](https://github.com/elastic/elasticsearch/pull/113216) +Snapshot/Restore: +* Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) -Mapping -: * Deprecate `_source.mode` in mappings [#116689](https://github.com/elastic/elasticsearch/pull/116689) \ No newline at end of file From 0213a71e69e7e460b918c62044f0bf9132e94e22 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 5 Mar 2025 22:38:45 +0000 Subject: [PATCH 02/34] [CI] Auto commit changes from spotless --- .../gradle/internal/release/ReleaseToolsPlugin.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index 7bfc8f70470f8..f04a2e74cc5c3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -81,17 +81,13 @@ public void apply(Project project) { task.setReleaseNotesIndexFile(projectDirectory.file("docs/reference/release-notes.asciidoc")); task.setReleaseNotesTemplate(projectDirectory.file(RESOURCES + "templates/release-notes.md")); - task.setReleaseNotesFile( - projectDirectory.file("docs/release-notes/index.md") - ); + task.setReleaseNotesFile(projectDirectory.file("docs/release-notes/index.md")); task.setReleaseHighlightsTemplate(projectDirectory.file(RESOURCES + "templates/release-highlights.asciidoc")); task.setReleaseHighlightsFile(projectDirectory.file("docs/reference/release-notes/highlights.asciidoc")); task.setBreakingChangesTemplate(projectDirectory.file(RESOURCES + "templates/breaking-changes.md")); - task.setBreakingChangesMigrationFile( - projectDirectory.file("docs/release-notes/breaking-changes.md") - ); + task.setBreakingChangesMigrationFile(projectDirectory.file("docs/release-notes/breaking-changes.md")); task.setMigrationIndexTemplate(projectDirectory.file(RESOURCES + "templates/migration-index.asciidoc")); task.setMigrationIndexFile(projectDirectory.file("docs/reference/migration/index.asciidoc")); From ca352dd5b24d7ce8fe99b9c7cc2b12127e156f58 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 6 Mar 2025 15:30:13 -0500 Subject: [PATCH 03/34] More work on moving release notes automation to markdown --- .../release/GenerateReleaseNotesTask.java | 29 + .../internal/release/ReleaseToolsPlugin.java | 5 + .../resources/templates/breaking-changes.md | 28 +- .../main/resources/templates/deprecations.md | 39 ++ docs/release-notes/breaking-changes.md | 84 +-- docs/release-notes/deprecations.md | 17 +- docs/release-notes/index.md | 508 +----------------- 7 files changed, 105 insertions(+), 605 deletions(-) create mode 100644 build-tools-internal/src/main/resources/templates/deprecations.md diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index 668f5730b3957..70cb214fc8c12 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -57,12 +57,14 @@ public class GenerateReleaseNotesTask extends DefaultTask { private final RegularFileProperty releaseNotesTemplate; private final RegularFileProperty releaseHighlightsTemplate; private final RegularFileProperty breakingChangesTemplate; + private final RegularFileProperty deprecationsTemplate; private final RegularFileProperty migrationIndexTemplate; private final RegularFileProperty releaseNotesIndexFile; private final RegularFileProperty releaseNotesFile; private final RegularFileProperty releaseHighlightsFile; private final RegularFileProperty breakingChangesMigrationFile; + private final RegularFileProperty deprecationsFile; private final RegularFileProperty migrationIndexFile; private final GitWrapper gitWrapper; @@ -75,12 +77,14 @@ public GenerateReleaseNotesTask(ObjectFactory objectFactory, ExecOperations exec releaseNotesTemplate = objectFactory.fileProperty(); releaseHighlightsTemplate = objectFactory.fileProperty(); breakingChangesTemplate = objectFactory.fileProperty(); + deprecationsTemplate = objectFactory.fileProperty(); migrationIndexTemplate = objectFactory.fileProperty(); releaseNotesIndexFile = objectFactory.fileProperty(); releaseNotesFile = objectFactory.fileProperty(); releaseHighlightsFile = objectFactory.fileProperty(); breakingChangesMigrationFile = objectFactory.fileProperty(); + deprecationsFile = objectFactory.fileProperty(); migrationIndexFile = objectFactory.fileProperty(); gitWrapper = new GitWrapper(execOperations); @@ -146,6 +150,13 @@ public void executeTask() throws IOException { changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) ); + ReleaseNotesGenerator.update( + this.deprecationsTemplate.get().getAsFile(), + this.deprecationsFile.get().getAsFile(), + qualifiedVersion, + changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) + ); + LOGGER.info("Updating migration/index..."); MigrationIndexGenerator.update( getMinorVersions(versions), @@ -354,6 +365,15 @@ public void setBreakingChangesTemplate(RegularFile file) { this.breakingChangesTemplate.set(file); } + @InputFile + public RegularFileProperty getDeprecationsTemplate() { + return deprecationsTemplate; + } + + public void setDeprecationsTemplate(RegularFile file) { + this.deprecationsTemplate.set(file); + } + @InputFile public RegularFileProperty getMigrationIndexTemplate() { return migrationIndexTemplate; @@ -399,6 +419,15 @@ public void setBreakingChangesMigrationFile(RegularFile file) { this.breakingChangesMigrationFile.set(file); } + @OutputFile + public RegularFileProperty getDeprecationsFile() { + return deprecationsFile; + } + + public void setDeprecationsFile(RegularFile file) { + this.deprecationsFile.set(file); + } + @OutputFile public RegularFileProperty getMigrationIndexFile() { return migrationIndexFile; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index 7bfc8f70470f8..8d834c40efca6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -95,6 +95,11 @@ public void apply(Project project) { task.setMigrationIndexTemplate(projectDirectory.file(RESOURCES + "templates/migration-index.asciidoc")); task.setMigrationIndexFile(projectDirectory.file("docs/reference/migration/index.asciidoc")); + task.setDeprecationsTemplate(projectDirectory.file(RESOURCES + "templates/deprecations.md")); + task.setDeprecationsFile( + projectDirectory.file("docs/release-notes/deprecations.md") + ); + task.dependsOn(validateChangelogsTask); }; diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.md b/build-tools-internal/src/main/resources/templates/breaking-changes.md index 066a6b5ff27ac..67bf5c0283831 100644 --- a/build-tools-internal/src/main/resources/templates/breaking-changes.md +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.md @@ -14,17 +14,23 @@ To learn how to upgrade, check out . ## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-breaking-changes] **Release date:** April 01, 2025 -<% for (team in changelogsByTypeByArea['breaking'].keySet()) { - print "\n${team}:\n"; +<% + if (!changelogsByTypeByArea['breaking']) { + print "\nNo breaking changes in this version.\n" + } else { + for (team in (changelogsByTypeByArea['breaking'] ?: [:]).keySet()) { + print "\n${team}:\n"; - for (change in changelogsByTypeByArea['breaking'][team]) { - print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" - if (change.issues != null && change.issues.empty == false) { - print change.issues.size() == 1 ? " (issue: " : " (issues: " - print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") - print ")" + for (change in changelogsByTypeByArea['breaking'][team]) { + print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" + if (change.issues != null && change.issues.empty == false) { + print change.issues.size() == 1 ? " (issue: " : " (issues: " + print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") + print ")" + } + print "\n" + } } - print "\n" + + print "\n\n" } -} -print "\n\n" diff --git a/build-tools-internal/src/main/resources/templates/deprecations.md b/build-tools-internal/src/main/resources/templates/deprecations.md new file mode 100644 index 0000000000000..b2b5e35f8d552 --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/deprecations.md @@ -0,0 +1,39 @@ +--- +navigation_title: "Elasticsearch" +--- + +# {{es}} deprecations [elasticsearch-deprecations] +Review the deprecated functionality for your {{es}} version. While deprecations have no immediate impact, we strongly encourage you update your implementation after you upgrade. + +To learn how to upgrade, check out . + +To give you insight into what deprecated features you’re using, {{es}}: + +* Returns a `Warn` HTTP header whenever you submit a request that uses deprecated functionality. +* [Logs deprecation warnings](docs-content://deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md#deprecation-logging) when deprecated functionality is used. +* [Provides a deprecation info API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-migration-deprecations) that scans a cluster’s configuration and mappings for deprecated functionality. + +% ## Next version [elasticsearch-nextversion-deprecations] +% **Release date:** Month day, year + +## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-deprecations] +**Release date:** April 01, 2025 +<% + if (!changelogsByTypeByArea['deprecations']) { + print "\nNo deprecations in this version.\n" + } else { + for (team in (changelogsByTypeByArea['deprecations'] ?: [:]).keySet()) { + print "\n${team}:\n"; + + for (change in changelogsByTypeByArea['deprecations'][team]) { + print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" + if (change.issues != null && change.issues.empty == false) { + print change.issues.size() == 1 ? " (issue: " : " (issues: " + print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") + print ")" + } + print "\n" + } + } + print "\n\n" + } diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index d3754dab5e43b..c8456bfca81e6 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -15,89 +15,7 @@ To learn how to upgrade, check out . ## 9.1.0 [elasticsearch-910-breaking-changes] **Release date:** April 01, 2025 -Aggregations: -* Remove date histogram boolean support [#118484](https://github.com/elastic/elasticsearch/pull/118484) - -Allocation: -* Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831) -* Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting [#114207](https://github.com/elastic/elasticsearch/pull/114207) -* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: {es-issue}88978[#88978]) - -Analysis: -* Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146) -* The 'german2' stemmer is now an alias for the 'german' snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614) -* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: {es-issue}113050[#113050]) -* The Korean dictionary for Nori has been updated [#114124](https://github.com/elastic/elasticsearch/pull/114124) - -Authentication: -* Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password -will prevent node from starting [#118366](https://github.com/elastic/elasticsearch/pull/118366) - -Cluster Coordination: -* Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903) - -EQL: -* Set allow_partial_search_results=true by default [#120267](https://github.com/elastic/elasticsearch/pull/120267) - -Extract&Transform: -* Restrict Connector APIs to manage/monitor_connector privileges [#119863](https://github.com/elastic/elasticsearch/pull/119863) - -Highlighting: -* Remove support for deprecated `force_source` highlighting parameter [#116943](https://github.com/elastic/elasticsearch/pull/116943) - -Indices APIs: -* Apply more strict parsing of actions in bulk API [#115923](https://github.com/elastic/elasticsearch/pull/115923) -* Remove deprecated local attribute from alias APIs [#115393](https://github.com/elastic/elasticsearch/pull/115393) -* Remove the ability to read frozen indices [#120108](https://github.com/elastic/elasticsearch/pull/120108) -* Remove unfreeze REST endpoint [#119227](https://github.com/elastic/elasticsearch/pull/119227) - -Infra/Core: -* Change Elasticsearch timeouts to 429 response instead of 5xx [#116026](https://github.com/elastic/elasticsearch/pull/116026) -* Limit `ByteSizeUnit` to 2 decimals [#120142](https://github.com/elastic/elasticsearch/pull/120142) -* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: {es-issue}104574[#104574]) -* Remove any references to org.elasticsearch.core.RestApiVersion#V_7 [#118103](https://github.com/elastic/elasticsearch/pull/118103) - -Infra/Logging: -* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: {es-issue}83251[#83251]) - -Infra/Metrics: -* Deprecated tracing.apm.* settings got removed. [#119926](https://github.com/elastic/elasticsearch/pull/119926) - -Infra/REST API: -* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: {es-issue}89387[#89387]) - -Ingest Node: -* Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077) -* Remove ignored fallback option on GeoIP processor [#116112](https://github.com/elastic/elasticsearch/pull/116112) - -Logs: -* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: {es-issue}106489[#106489]) - -Machine Learning: -* Disable machine learning on macOS x86_64 [#104125](https://github.com/elastic/elasticsearch/pull/104125) - -Mapping: -* Remove support for type, fields, `copy_to` and boost in metadata field definition [#118825](https://github.com/elastic/elasticsearch/pull/118825) -* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: {es-issue}118596[#118596]) - -Search: -* Adjust `random_score` default field to `_seq_no` field [#118671](https://github.com/elastic/elasticsearch/pull/118671) -* Change Semantic Text To Act Like A Normal Text Field [#120813](https://github.com/elastic/elasticsearch/pull/120813) -* Remove legacy params from range query [#116970](https://github.com/elastic/elasticsearch/pull/116970) - -Snapshot/Restore: -* Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting [#114202](https://github.com/elastic/elasticsearch/pull/114202) - TLS: -* Remove TLSv1.1 from default protocols [#121731](https://github.com/elastic/elasticsearch/pull/121731) - -Transform: -* Remove `data_frame_transforms` roles [#117519](https://github.com/elastic/elasticsearch/pull/117519) - -Vector Search: -* Remove old `_knn_search` tech preview API in v9 [#118104](https://github.com/elastic/elasticsearch/pull/118104) - -Watcher: -* Removing support for types field in watcher search [#120748](https://github.com/elastic/elasticsearch/pull/120748) +* Drop `TLS_RSA` cipher support for JDK 24 [#123600](https://github.com/elastic/elasticsearch/pull/123600) diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index a911bfc677565..2e2269249ad41 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -5,7 +5,7 @@ navigation_title: "Elasticsearch" # {{es}} deprecations [elasticsearch-deprecations] Review the deprecated functionality for your {{es}} version. While deprecations have no immediate impact, we strongly encourage you update your implementation after you upgrade. -To learn how to upgrade, check out . +To learn how to upgrade, check out . To give you insight into what deprecated features you’re using, {{es}}: @@ -13,19 +13,12 @@ To give you insight into what deprecated features you’re using, {{es}}: * [Logs deprecation warnings](docs-content://deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md#deprecation-logging) when deprecated functionality is used. * [Provides a deprecation info API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-migration-deprecations) that scans a cluster’s configuration and mappings for deprecated functionality. -% ## Next version +% ## Next version [elasticsearch-nextversion-deprecations] % **Release date:** Month day, year -## 9.0.0 [elasticsearch-900-deprecations] -**Release date:** March 25, 2025 +## 9.1.0 [elasticsearch-910-deprecations] +**Release date:** April 01, 2025 -Ingest Node -: * Fix `_type` deprecation on simulate pipeline API [#116259](https://github.com/elastic/elasticsearch/pull/116259) +No deprecations in this version. -Machine Learning -: * [Inference API] Deprecate elser service [#113216](https://github.com/elastic/elasticsearch/pull/113216) - - -Mapping -: * Deprecate `_source.mode` in mappings [#116689](https://github.com/elastic/elasticsearch/pull/116689) diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 7f530eb4c80b5..5294609c89299 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -29,400 +29,80 @@ To check for security updates, go to [Security announcements for the Elastic sta ### Breaking changes [elasticsearch-910-breaking] -Aggregations: -* Remove date histogram boolean support [#118484](https://github.com/elastic/elasticsearch/pull/118484) - -Allocation: -* Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831) -* Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting [#114207](https://github.com/elastic/elasticsearch/pull/114207) -* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: {es-issue}88978[#88978]) - -Analysis: -* Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146) -* The 'german2' stemmer is now an alias for the 'german' snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614) -* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: {es-issue}113050[#113050]) -* The Korean dictionary for Nori has been updated [#114124](https://github.com/elastic/elasticsearch/pull/114124) - -Authentication: -* Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password -will prevent node from starting [#118366](https://github.com/elastic/elasticsearch/pull/118366) - -Cluster Coordination: -* Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903) - -EQL: -* Set allow_partial_search_results=true by default [#120267](https://github.com/elastic/elasticsearch/pull/120267) - -Extract&Transform: -* Restrict Connector APIs to manage/monitor_connector privileges [#119863](https://github.com/elastic/elasticsearch/pull/119863) - -Highlighting: -* Remove support for deprecated `force_source` highlighting parameter [#116943](https://github.com/elastic/elasticsearch/pull/116943) - -Indices APIs: -* Apply more strict parsing of actions in bulk API [#115923](https://github.com/elastic/elasticsearch/pull/115923) -* Remove deprecated local attribute from alias APIs [#115393](https://github.com/elastic/elasticsearch/pull/115393) -* Remove the ability to read frozen indices [#120108](https://github.com/elastic/elasticsearch/pull/120108) -* Remove unfreeze REST endpoint [#119227](https://github.com/elastic/elasticsearch/pull/119227) - -Infra/Core: -* Change Elasticsearch timeouts to 429 response instead of 5xx [#116026](https://github.com/elastic/elasticsearch/pull/116026) -* Limit `ByteSizeUnit` to 2 decimals [#120142](https://github.com/elastic/elasticsearch/pull/120142) -* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: {es-issue}104574[#104574]) -* Remove any references to org.elasticsearch.core.RestApiVersion#V_7 [#118103](https://github.com/elastic/elasticsearch/pull/118103) - -Infra/Logging: -* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: {es-issue}83251[#83251]) - -Infra/Metrics: -* Deprecated tracing.apm.* settings got removed. [#119926](https://github.com/elastic/elasticsearch/pull/119926) - -Infra/REST API: -* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: {es-issue}89387[#89387]) - -Ingest Node: -* Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077) -* Remove ignored fallback option on GeoIP processor [#116112](https://github.com/elastic/elasticsearch/pull/116112) - -Logs: -* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: {es-issue}106489[#106489]) - -Machine Learning: -* Disable machine learning on macOS x86_64 [#104125](https://github.com/elastic/elasticsearch/pull/104125) - -Mapping: -* Remove support for type, fields, `copy_to` and boost in metadata field definition [#118825](https://github.com/elastic/elasticsearch/pull/118825) -* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: {es-issue}118596[#118596]) - -Search: -* Adjust `random_score` default field to `_seq_no` field [#118671](https://github.com/elastic/elasticsearch/pull/118671) -* Change Semantic Text To Act Like A Normal Text Field [#120813](https://github.com/elastic/elasticsearch/pull/120813) -* Remove legacy params from range query [#116970](https://github.com/elastic/elasticsearch/pull/116970) - -Snapshot/Restore: -* Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting [#114202](https://github.com/elastic/elasticsearch/pull/114202) - TLS: -* Remove TLSv1.1 from default protocols [#121731](https://github.com/elastic/elasticsearch/pull/121731) - -Transform: -* Remove `data_frame_transforms` roles [#117519](https://github.com/elastic/elasticsearch/pull/117519) - -Vector Search: -* Remove old `_knn_search` tech preview API in v9 [#118104](https://github.com/elastic/elasticsearch/pull/118104) - -Watcher: -* Removing support for types field in watcher search [#120748](https://github.com/elastic/elasticsearch/pull/120748) +* Drop `TLS_RSA` cipher support for JDK 24 [#123600](https://github.com/elastic/elasticsearch/pull/123600) ### Deprecations [elasticsearch-910-deprecation] -ES|QL: -* Drop support for brackets from METADATA syntax [#119846](https://github.com/elastic/elasticsearch/pull/119846) (issue: {es-issue}115401[#115401]) - -Ingest Node: -* Fix `_type` deprecation on simulate pipeline API [#116259](https://github.com/elastic/elasticsearch/pull/116259) - Machine Learning: * Add deprecation warning for flush API [#121667](https://github.com/elastic/elasticsearch/pull/121667) (issue: {es-issue}121506[#121506]) -* Removing index alias creation for deprecated transforms notification index [#117583](https://github.com/elastic/elasticsearch/pull/117583) -* [Inference API] Deprecate elser service [#113216](https://github.com/elastic/elasticsearch/pull/113216) - -Rollup: -* Emit deprecation warning when executing one of the rollup APIs [#113131](https://github.com/elastic/elasticsearch/pull/113131) Search: * Deprecate Behavioral Analytics CRUD apis [#122960](https://github.com/elastic/elasticsearch/pull/122960) -Security: -* Deprecate certificate based remote cluster security model [#120806](https://github.com/elastic/elasticsearch/pull/120806) - ### Features and enhancements [elasticsearch-910-features-enhancements] Allocation: -* Add a not-master state for desired balance [#116904](https://github.com/elastic/elasticsearch/pull/116904) * Introduce `AllocationBalancingRoundSummaryService` [#120957](https://github.com/elastic/elasticsearch/pull/120957) -* Only publish desired balance gauges on master [#115383](https://github.com/elastic/elasticsearch/pull/115383) -* Reset relocation/allocation failure counter on node join/shutdown [#119968](https://github.com/elastic/elasticsearch/pull/119968) - -Authentication: -* Allow `SSHA-256` for API key credential hash [#120997](https://github.com/elastic/elasticsearch/pull/120997) Authorization: -* Allow kibana_system user to manage .reindexed-v8-internal.alerts indices [#118959](https://github.com/elastic/elasticsearch/pull/118959) * Do not fetch reserved roles from native store when Get Role API is called [#121971](https://github.com/elastic/elasticsearch/pull/121971) -* Grant necessary Kibana application privileges to `reporting_user` role [#118058](https://github.com/elastic/elasticsearch/pull/118058) -* Make reserved built-in roles queryable [#117581](https://github.com/elastic/elasticsearch/pull/117581) -* [Security Solution] Add `create_index` to `kibana_system` role for index/DS `.logs-endpoint.action.responses-*` [#115241](https://github.com/elastic/elasticsearch/pull/115241) -* [Security Solution] allows `kibana_system` user to manage .reindexed-v8-* Security Solution indices [#119054](https://github.com/elastic/elasticsearch/pull/119054) - -CCS: -* Resolve/cluster allows querying for cluster info only (no index expression required) [#119898](https://github.com/elastic/elasticsearch/pull/119898) - -CRUD: -* Metrics for indexing failures due to version conflicts [#119067](https://github.com/elastic/elasticsearch/pull/119067) -* Suppress merge-on-recovery for older indices [#113462](https://github.com/elastic/elasticsearch/pull/113462) - -Cluster Coordination: -* Include `clusterApplyListener` in long cluster apply warnings [#120087](https://github.com/elastic/elasticsearch/pull/120087) Data streams: -* Add action to create index from a source index [#118890](https://github.com/elastic/elasticsearch/pull/118890) -* Add index and reindex request settings to speed up reindex [#119780](https://github.com/elastic/elasticsearch/pull/119780) -* Add rest endpoint for `create_from_source_index` [#119250](https://github.com/elastic/elasticsearch/pull/119250) -* Add sanity check to `ReindexDatastreamIndexAction` [#120231](https://github.com/elastic/elasticsearch/pull/120231) -* Adding a migration reindex cancel API [#118291](https://github.com/elastic/elasticsearch/pull/118291) -* Adding get migration reindex status [#118267](https://github.com/elastic/elasticsearch/pull/118267) -* Consistent mapping for OTel log and event bodies [#120547](https://github.com/elastic/elasticsearch/pull/120547) -* Filter deprecated settings when making dest index [#120163](https://github.com/elastic/elasticsearch/pull/120163) -* Ignore closed indices for reindex [#120244](https://github.com/elastic/elasticsearch/pull/120244) -* Improve how reindex data stream index action handles api blocks [#120084](https://github.com/elastic/elasticsearch/pull/120084) -* Initial work on `ReindexDatastreamIndexAction` [#116996](https://github.com/elastic/elasticsearch/pull/116996) -* Make `requests_per_second` configurable to throttle reindexing [#120207](https://github.com/elastic/elasticsearch/pull/120207) -* Optimized index sorting for OTel logs [#119504](https://github.com/elastic/elasticsearch/pull/119504) -* Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As A Warning [#120629](https://github.com/elastic/elasticsearch/pull/120629) -* Update data stream deprecations warnings to new format and filter searchable snapshots from response [#118562](https://github.com/elastic/elasticsearch/pull/118562) - -Distributed: -* Make various alias retrieval APIs wait for cluster to unblock [#117230](https://github.com/elastic/elasticsearch/pull/117230) -* Metrics for incremental bulk splits [#116765](https://github.com/elastic/elasticsearch/pull/116765) -* Use Azure blob batch API to delete blobs in batches [#114566](https://github.com/elastic/elasticsearch/pull/114566) - -EQL: -* Add support for partial shard results [#116388](https://github.com/elastic/elasticsearch/pull/116388) -* Optional named arguments for function in map [#118619](https://github.com/elastic/elasticsearch/pull/118619) +* Add index mode to get data stream API [#122486](https://github.com/elastic/elasticsearch/pull/122486) ES|QL: -* Add ES|QL cross-cluster query telemetry collection [#119474](https://github.com/elastic/elasticsearch/pull/119474) -* Add a `LicenseAware` interface for licensed Nodes [#118931](https://github.com/elastic/elasticsearch/pull/118931) (issue: {es-issue}117405[#117405]) -* Add a `PostAnalysisAware,` distribute verification [#119798](https://github.com/elastic/elasticsearch/pull/119798) -* Add a standard deviation aggregating function: STD_DEV [#116531](https://github.com/elastic/elasticsearch/pull/116531) -* Add cluster level reduction [#117731](https://github.com/elastic/elasticsearch/pull/117731) * Add initial grammar and changes for FORK [#121948](https://github.com/elastic/elasticsearch/pull/121948) -* Add nulls support to Categorize [#117655](https://github.com/elastic/elasticsearch/pull/117655) * Allow partial results in ES|QL [#121942](https://github.com/elastic/elasticsearch/pull/121942) * Allow skip shards with `_tier` and `_index` in ES|QL [#123728](https://github.com/elastic/elasticsearch/pull/123728) -* Async search responses have CCS metadata while searches are running [#117265](https://github.com/elastic/elasticsearch/pull/117265) -* Check for early termination in Driver [#118188](https://github.com/elastic/elasticsearch/pull/118188) -* Do not serialize `EsIndex` in plan [#119580](https://github.com/elastic/elasticsearch/pull/119580) -* ESQL - Add Match function options [#120360](https://github.com/elastic/elasticsearch/pull/120360) -* ESQL - Allow full text functions disjunctions for non-full text functions [#120291](https://github.com/elastic/elasticsearch/pull/120291) -* ESQL - Remove restrictions for disjunctions in full text functions [#118544](https://github.com/elastic/elasticsearch/pull/118544) -* ESQL - enabling scoring with METADATA `_score` [#113120](https://github.com/elastic/elasticsearch/pull/113120) -* ESQL Add esql hash function [#117989](https://github.com/elastic/elasticsearch/pull/117989) -* ESQL Support IN operator for Date nanos [#119772](https://github.com/elastic/elasticsearch/pull/119772) (issue: {es-issue}118578[#118578]) -* ESQL: CATEGORIZE as a `BlockHash` [#114317](https://github.com/elastic/elasticsearch/pull/114317) -* ESQL: Enable async get to support formatting [#111104](https://github.com/elastic/elasticsearch/pull/111104) (issue: {es-issue}110926[#110926]) -* ESQL: Enterprise license enforcement for CCS [#118102](https://github.com/elastic/elasticsearch/pull/118102) -* ES|QL: Partial result on demand for async queries [#118122](https://github.com/elastic/elasticsearch/pull/118122) -* Enable KQL function as a tech preview [#119730](https://github.com/elastic/elasticsearch/pull/119730) -* Enable LOOKUP JOIN in non-snapshot builds [#121193](https://github.com/elastic/elasticsearch/pull/121193) (issue: {es-issue}121185[#121185]) -* Enable node-level reduction by default [#119621](https://github.com/elastic/elasticsearch/pull/119621) -* Enable physical plan verification [#118114](https://github.com/elastic/elasticsearch/pull/118114) -* Ensure cluster string could be quoted [#120355](https://github.com/elastic/elasticsearch/pull/120355) -* Esql - Support date nanos in date extract function [#120727](https://github.com/elastic/elasticsearch/pull/120727) (issue: {es-issue}110000[#110000]) -* Esql - support date nanos in date format function [#120143](https://github.com/elastic/elasticsearch/pull/120143) (issue: {es-issue}109994[#109994]) -* Esql Support date nanos on date diff function [#120645](https://github.com/elastic/elasticsearch/pull/120645) (issue: {es-issue}109999[#109999]) -* Esql bucket function for date nanos [#118474](https://github.com/elastic/elasticsearch/pull/118474) (issue: {es-issue}118031[#118031]) -* Esql compare nanos and millis [#118027](https://github.com/elastic/elasticsearch/pull/118027) (issue: {es-issue}116281[#116281]) -* Esql implicit casting for date nanos [#118697](https://github.com/elastic/elasticsearch/pull/118697) (issue: {es-issue}118476[#118476]) -* Expand type compatibility for match function and operator [#117555](https://github.com/elastic/elasticsearch/pull/117555) -* Extend `TranslationAware` to all pushable expressions [#120192](https://github.com/elastic/elasticsearch/pull/120192) * Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: {es-issue}122967[#122967]) -* Hash functions [#118938](https://github.com/elastic/elasticsearch/pull/118938) -* Implement a `MetricsAware` interface [#121074](https://github.com/elastic/elasticsearch/pull/121074) * Implement runtime skip_unavailable=true [#121240](https://github.com/elastic/elasticsearch/pull/121240) * Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) +* Introduce `allow_partial_results` setting in ES|QL [#122890](https://github.com/elastic/elasticsearch/pull/122890) * Introduce a pre-mapping logical plan processing step [#121260](https://github.com/elastic/elasticsearch/pull/121260) -* LOOKUP JOIN using field-caps for field mapping [#117246](https://github.com/elastic/elasticsearch/pull/117246) -* Lookup join on multiple join fields not yet supported [#118858](https://github.com/elastic/elasticsearch/pull/118858) -* Move scoring in ES|QL out of snapshot [#120354](https://github.com/elastic/elasticsearch/pull/120354) -* Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` [#119889](https://github.com/elastic/elasticsearch/pull/119889) -* Push down filter passed lookup join [#118410](https://github.com/elastic/elasticsearch/pull/118410) * Render `aggregate_metric_double` [#122660](https://github.com/elastic/elasticsearch/pull/122660) -* Resume Driver on cancelled or early finished [#120020](https://github.com/elastic/elasticsearch/pull/120020) * Retry ES|QL node requests on shard level failures [#120774](https://github.com/elastic/elasticsearch/pull/120774) -* Rewrite TO_UPPER/TO_LOWER comparisons [#118870](https://github.com/elastic/elasticsearch/pull/118870) (issue: {es-issue}118304[#118304]) -* ST_EXTENT aggregation [#117451](https://github.com/elastic/elasticsearch/pull/117451) (issue: {es-issue}104659[#104659]) -* ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape [#118802](https://github.com/elastic/elasticsearch/pull/118802) -* Smarter field caps with subscribable listener [#116755](https://github.com/elastic/elasticsearch/pull/116755) -* Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions [#116964](https://github.com/elastic/elasticsearch/pull/116964) (issue: {es-issue}104875[#104875]) * Support partial results in CCS in ES|QL [#122708](https://github.com/elastic/elasticsearch/pull/122708) -* Support partial sort fields in TopN pushdown [#116043](https://github.com/elastic/elasticsearch/pull/116043) (issue: {es-issue}114515[#114515]) -* Support some stats on aggregate_metric_double [#120343](https://github.com/elastic/elasticsearch/pull/120343) (issue: {es-issue}110649[#110649]) * Support subset of metrics in aggregate metric double [#121805](https://github.com/elastic/elasticsearch/pull/121805) -* Take named parameters for identifier and pattern out of snapshot [#121850](https://github.com/elastic/elasticsearch/pull/121850) -* Term query for ES|QL [#117359](https://github.com/elastic/elasticsearch/pull/117359) -* Update grammar to rely on `indexPattern` instead of identifier in join target [#120494](https://github.com/elastic/elasticsearch/pull/120494) -* `_score` should not be a reserved attribute in ES|QL [#118435](https://github.com/elastic/elasticsearch/pull/118435) (issue: {es-issue}118460[#118460]) - -Engine: -* Defer unpromotable shard refreshes until index refresh blocks are cleared [#120642](https://github.com/elastic/elasticsearch/pull/120642) -* POC mark read-only [#119743](https://github.com/elastic/elasticsearch/pull/119743) - -Experiences: -* Integrate IBM watsonx to Inference API for re-ranking task [#117176](https://github.com/elastic/elasticsearch/pull/117176) - -Extract&Transform: -* [Connector API] Support hard deletes with new URL param in delete endpoint [#120200](https://github.com/elastic/elasticsearch/pull/120200) -* [Connector API] Support soft-deletes of connectors [#118669](https://github.com/elastic/elasticsearch/pull/118669) -* [Connector APIs] Enforce index prefix for managed connectors [#117778](https://github.com/elastic/elasticsearch/pull/117778) - -Geo: -* Optimize indexing points with index and doc values set to true [#120271](https://github.com/elastic/elasticsearch/pull/120271) Health: * Add health indicator impact to `HealthPeriodicLogger` [#122390](https://github.com/elastic/elasticsearch/pull/122390) -* Increase `replica_unassigned_buffer_time` default from 3s to 5s [#112834](https://github.com/elastic/elasticsearch/pull/112834) - -Highlighting: -* Add Highlighter for Semantic Text Fields [#118064](https://github.com/elastic/elasticsearch/pull/118064) ILM+SLM: -* Add a `replicate_for` option to the ILM `searchable_snapshot` action [#119003](https://github.com/elastic/elasticsearch/pull/119003) * Improve SLM Health Indicator to cover missing snapshot [#121370](https://github.com/elastic/elasticsearch/pull/121370) -Indices APIs: -* Add `remove_index_block` arg to `_create_from` api [#120548](https://github.com/elastic/elasticsearch/pull/120548) -* Remove index blocks by default in `create_from` [#120643](https://github.com/elastic/elasticsearch/pull/120643) -* Run `TransportGetComponentTemplateAction` on local node [#116868](https://github.com/elastic/elasticsearch/pull/116868) -* Run `TransportGetComposableIndexTemplate` on local node [#119830](https://github.com/elastic/elasticsearch/pull/119830) -* Run `TransportGetIndexTemplateAction` on local node [#119837](https://github.com/elastic/elasticsearch/pull/119837) -* introduce new categories for deprecated resources in deprecation API [#120505](https://github.com/elastic/elasticsearch/pull/120505) - Inference: -* Add version prefix to Inference Service API path [#117095](https://github.com/elastic/elasticsearch/pull/117095) -* Remove Elastic Inference Service feature flag and deprecated setting [#120842](https://github.com/elastic/elasticsearch/pull/120842) -* Update sparse text embeddings API route for Inference Service [#118025](https://github.com/elastic/elasticsearch/pull/118025) -* [Elastic Inference Service] Add ElasticInferenceService Unified ChatCompletions Integration [#118871](https://github.com/elastic/elasticsearch/pull/118871) * [Inference API] Rename `model_id` prop to model in EIS sparse inference request body [#122272](https://github.com/elastic/elasticsearch/pull/122272) -Infra/CLI: -* Strengthen encryption for elasticsearch-keystore tool to AES 256 [#119749](https://github.com/elastic/elasticsearch/pull/119749) - -Infra/Circuit Breakers: -* Add link to Circuit Breaker "Data too large" exception message [#113561](https://github.com/elastic/elasticsearch/pull/113561) - Infra/Core: -* Add support for specifying reindexing script for system index migration [#119001](https://github.com/elastic/elasticsearch/pull/119001) -* Change default Docker image to be based on UBI minimal instead of Ubuntu [#116739](https://github.com/elastic/elasticsearch/pull/116739) * Improve size limiting string message [#122427](https://github.com/elastic/elasticsearch/pull/122427) -* Infrastructure for assuming cluster features in the next major version [#118143](https://github.com/elastic/elasticsearch/pull/118143) - -Infra/Metrics: -* Add `ensureGreen` test method for use with `adminClient` [#113425](https://github.com/elastic/elasticsearch/pull/113425) - -Infra/REST API: -* A new query parameter `?include_source_on_error` was added for create / index, update and bulk REST APIs to control -if to include the document source in the error response in case of parsing errors. The default value is `true`. [#120725](https://github.com/elastic/elasticsearch/pull/120725) - -Infra/Scripting: -* Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts [#114002](https://github.com/elastic/elasticsearch/pull/114002) - -Infra/Settings: -* Introduce `IndexSettingDeprecatedInV8AndRemovedInV9` Setting property [#120334](https://github.com/elastic/elasticsearch/pull/120334) -* Run `TransportClusterGetSettingsAction` on local node [#119831](https://github.com/elastic/elasticsearch/pull/119831) Ingest Node: * Allow setting the `type` in the reroute processor [#122409](https://github.com/elastic/elasticsearch/pull/122409) (issue: {es-issue}121553[#121553]) -* Optimize `IngestCtxMap` construction [#120833](https://github.com/elastic/elasticsearch/pull/120833) -* Optimize `IngestDocMetadata` `isAvailable` [#120753](https://github.com/elastic/elasticsearch/pull/120753) -* Optimize `IngestDocument` `FieldPath` allocation [#120573](https://github.com/elastic/elasticsearch/pull/120573) -* Optimize some per-document hot paths in the geoip processor [#120824](https://github.com/elastic/elasticsearch/pull/120824) -* Returning ignored fields in the simulate ingest API [#117214](https://github.com/elastic/elasticsearch/pull/117214) -* Run `GetPipelineTransportAction` on local node [#120445](https://github.com/elastic/elasticsearch/pull/120445) * Run `TransportEnrichStatsAction` on local node [#121256](https://github.com/elastic/elasticsearch/pull/121256) -* Run `TransportGetEnrichPolicyAction` on local node [#121124](https://github.com/elastic/elasticsearch/pull/121124) -* Run template simulation actions on local node [#120038](https://github.com/elastic/elasticsearch/pull/120038) - -License: -* Bump `TrialLicenseVersion` to allow starting new trial on 9.0 [#120198](https://github.com/elastic/elasticsearch/pull/120198) - -Logs: -* Add LogsDB option to route on sort fields [#116687](https://github.com/elastic/elasticsearch/pull/116687) -* Add a new index setting to skip recovery source when synthetic source is enabled [#114618](https://github.com/elastic/elasticsearch/pull/114618) -* Configure index sorting through index settings for logsdb [#118968](https://github.com/elastic/elasticsearch/pull/118968) (issue: {es-issue}118686[#118686]) -* Optimize loading mappings when determining synthetic source usage and whether host.name can be sorted on. [#120055](https://github.com/elastic/elasticsearch/pull/120055) Machine Learning: -* Add DeBERTa-V2/V3 tokenizer [#111852](https://github.com/elastic/elasticsearch/pull/111852) -* Add Inference Unified API for chat completions for OpenAI [#117589](https://github.com/elastic/elasticsearch/pull/117589) -* Add Jina AI API to do inference for Embedding and Rerank models [#118652](https://github.com/elastic/elasticsearch/pull/118652) -* Add enterprise license check for Inference API actions [#119893](https://github.com/elastic/elasticsearch/pull/119893) -* Adding chunking settings to `IbmWatsonxService` [#114914](https://github.com/elastic/elasticsearch/pull/114914) -* Adding default endpoint for Elastic Rerank [#117939](https://github.com/elastic/elasticsearch/pull/117939) * Adding elser default endpoint for EIS [#122066](https://github.com/elastic/elasticsearch/pull/122066) -* Adding endpoint creation validation for all task types to remaining services [#115020](https://github.com/elastic/elasticsearch/pull/115020) * Adding endpoint creation validation to `ElasticInferenceService` [#117642](https://github.com/elastic/elasticsearch/pull/117642) * Adding integration for VoyageAI embeddings and rerank models [#122134](https://github.com/elastic/elasticsearch/pull/122134) * Adding support for binary embedding type to Cohere service embedding type [#120751](https://github.com/elastic/elasticsearch/pull/120751) * Adding support for specifying embedding type to Jina AI service settings [#121548](https://github.com/elastic/elasticsearch/pull/121548) -* Check for presence of error object when validating streaming responses from integrations in the inference API [#118375](https://github.com/elastic/elasticsearch/pull/118375) * ES|QL `change_point` processing command [#120998](https://github.com/elastic/elasticsearch/pull/120998) -* ES|QL categorize with multiple groupings [#118173](https://github.com/elastic/elasticsearch/pull/118173) -* Ignore failures from renormalizing buckets in read-only index [#118674](https://github.com/elastic/elasticsearch/pull/118674) -* Inference duration and error metrics [#115876](https://github.com/elastic/elasticsearch/pull/115876) -* Migrate stream to core error parsing [#120722](https://github.com/elastic/elasticsearch/pull/120722) -* Remove all mentions of eis and gateway and deprecate flags that do [#116692](https://github.com/elastic/elasticsearch/pull/116692) -* Remove deprecated sort from reindex operation within dataframe analytics procedure [#117606](https://github.com/elastic/elasticsearch/pull/117606) -* Retry on `ClusterBlockException` on transform destination index [#118194](https://github.com/elastic/elasticsearch/pull/118194) -* Support mTLS for the Elastic Inference Service integration inside the inference API [#119679](https://github.com/elastic/elasticsearch/pull/119679) -* [Inference API] Add node-local rate limiting for the inference API [#120400](https://github.com/elastic/elasticsearch/pull/120400) -* [Inference API] fix spell words: covertToString to convertToString [#119922](https://github.com/elastic/elasticsearch/pull/119922) Mapping: -* Add Optional Source Filtering to Source Loaders [#113827](https://github.com/elastic/elasticsearch/pull/113827) -* Add option to store `sparse_vector` outside `_source` [#117917](https://github.com/elastic/elasticsearch/pull/117917) +* Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. [#122615](https://github.com/elastic/elasticsearch/pull/122615) (issue: {es-issue}116726[#116726]) * Enable the use of nested field type with index.mode=time_series [#122224](https://github.com/elastic/elasticsearch/pull/122224) (issue: {es-issue}120874[#120874]) * Improved error message when index field type is invalid [#122860](https://github.com/elastic/elasticsearch/pull/122860) * Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields [#119546](https://github.com/elastic/elasticsearch/pull/119546) * Store arrays offsets for ip fields natively with synthetic source [#122999](https://github.com/elastic/elasticsearch/pull/122999) * Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. [#113757](https://github.com/elastic/elasticsearch/pull/113757) * Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields [#122637](https://github.com/elastic/elasticsearch/pull/122637) +* Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields [#124050](https://github.com/elastic/elasticsearch/pull/124050) * Use `FallbackSyntheticSourceBlockLoader` for number fields [#122280](https://github.com/elastic/elasticsearch/pull/122280) -Network: -* Allow http unsafe buffers by default [#116115](https://github.com/elastic/elasticsearch/pull/116115) -* Http stream activity tracker and exceptions handling [#119564](https://github.com/elastic/elasticsearch/pull/119564) -* Remove HTTP content copies [#117303](https://github.com/elastic/elasticsearch/pull/117303) -* `ConnectTransportException` returns retryable BAD_GATEWAY [#118681](https://github.com/elastic/elasticsearch/pull/118681) (issue: {es-issue}118320[#118320]) - -Ranking: -* Add a generic `rescorer` retriever based on the search request's rescore functionality [#118585](https://github.com/elastic/elasticsearch/pull/118585) (issue: {es-issue}118327[#118327]) -* Set default reranker for text similarity reranker to Elastic reranker [#120551](https://github.com/elastic/elasticsearch/pull/120551) - -Recovery: -* Allow archive and searchable snapshots indices in N-2 version [#118941](https://github.com/elastic/elasticsearch/pull/118941) -* Trigger merges after recovery [#113102](https://github.com/elastic/elasticsearch/pull/113102) - -Reindex: -* Change Reindexing metrics unit from millis to seconds [#115721](https://github.com/elastic/elasticsearch/pull/115721) - -Relevance: -* Add Multi-Field Support for Semantic Text Fields [#120128](https://github.com/elastic/elasticsearch/pull/120128) - Search: * Account for the `SearchHit` source in circuit breaker [#121920](https://github.com/elastic/elasticsearch/pull/121920) (issue: {es-issue}89656[#89656]) -* Add match support for `semantic_text` fields [#117839](https://github.com/elastic/elasticsearch/pull/117839) -* Add support for `sparse_vector` queries against `semantic_text` fields [#118617](https://github.com/elastic/elasticsearch/pull/118617) -* Add support for knn vector queries on `semantic_text` fields [#119011](https://github.com/elastic/elasticsearch/pull/119011) -* Adding linear retriever to support weighted sums of sub-retrievers [#120222](https://github.com/elastic/elasticsearch/pull/120222) -* Address and remove any references of RestApiVersion version 7 [#117572](https://github.com/elastic/elasticsearch/pull/117572) -* Feat: add a user-configurable timeout parameter to the `_resolve/cluster` API [#120542](https://github.com/elastic/elasticsearch/pull/120542) -* Make semantic text part of the text family [#119792](https://github.com/elastic/elasticsearch/pull/119792) -* Only aggregations require at least one shard request [#115314](https://github.com/elastic/elasticsearch/pull/115314) * Optionally allow text similarity reranking to fail [#121784](https://github.com/elastic/elasticsearch/pull/121784) -* Prevent data nodes from sending stack traces to coordinator when `error_trace=false` [#118266](https://github.com/elastic/elasticsearch/pull/118266) -* Propagate status codes from shard failures appropriately [#118016](https://github.com/elastic/elasticsearch/pull/118016) (issue: {es-issue}118482[#118482]) - -Security: -* Add refresh `.security` index call between security migrations [#114879](https://github.com/elastic/elasticsearch/pull/114879) - -Snapshot/Restore: -* Add IMDSv2 support to `repository-s3` [#117748](https://github.com/elastic/elasticsearch/pull/117748) (issue: {es-issue}105135[#105135]) -* Expose operation and request counts separately in repository stats [#117530](https://github.com/elastic/elasticsearch/pull/117530) (issue: {es-issue}104443[#104443]) -* Retry `S3BlobContainer#getRegister` on all exceptions [#114813](https://github.com/elastic/elasticsearch/pull/114813) -* Retry internally when CAS upload is throttled [GCS] [#120250](https://github.com/elastic/elasticsearch/pull/120250) (issue: {es-issue}116546[#116546]) -* Track shard snapshot progress during node shutdown [#112567](https://github.com/elastic/elasticsearch/pull/112567) Stats: * Run XPack usage actions on local node [#122933](https://github.com/elastic/elasticsearch/pull/122933) @@ -430,216 +110,67 @@ Stats: Store: * Abort pending deletion on `IndicesService` close [#123569](https://github.com/elastic/elasticsearch/pull/123569) -Suggesters: -* Extensible Completion Postings Formats [#111494](https://github.com/elastic/elasticsearch/pull/111494) - -TSDB: -* Increase field limit for OTel metrics to 10 000 [#120591](https://github.com/elastic/elasticsearch/pull/120591) - -Transform: -* Add support for `extended_stats` [#120340](https://github.com/elastic/elasticsearch/pull/120340) -* Auto-migrate `max_page_search_size` [#119348](https://github.com/elastic/elasticsearch/pull/119348) -* Create upgrade mode [#117858](https://github.com/elastic/elasticsearch/pull/117858) -* Wait while index is blocked [#119542](https://github.com/elastic/elasticsearch/pull/119542) -* [Deprecation] Add `transform_ids` to outdated index [#120821](https://github.com/elastic/elasticsearch/pull/120821) - Vector Search: -* Add new experimental `rank_vectors` mapping for late-interaction second order ranking [#118804](https://github.com/elastic/elasticsearch/pull/118804) * Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields [#122381](https://github.com/elastic/elasticsearch/pull/122381) (issue: {es-issue}117274[#117274]) -* Even better(er) binary quantization [#117994](https://github.com/elastic/elasticsearch/pull/117994) -* KNN vector rescoring for quantized vectors [#116663](https://github.com/elastic/elasticsearch/pull/116663) -* Mark bbq indices as GA and add rolling upgrade integration tests [#121105](https://github.com/elastic/elasticsearch/pull/121105) -* Speed up bit compared with floats or bytes script operations [#117199](https://github.com/elastic/elasticsearch/pull/117199) Watcher: * Run `TransportGetWatcherSettingsAction` on local node [#122857](https://github.com/elastic/elasticsearch/pull/122857) ### Fixes [elasticsearch-910-fixes] -Aggregations: -* Disable concurrency when `top_hits` sorts on anything but `_score` [#123610](https://github.com/elastic/elasticsearch/pull/123610) -* Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations [#116174](https://github.com/elastic/elasticsearch/pull/116174) (issue: {es-issue}115777[#115777]) - Allocation: -* Deduplicate allocation stats calls [#123246](https://github.com/elastic/elasticsearch/pull/123246) * `DesiredBalanceReconciler` always returns `AllocationStats` [#122458](https://github.com/elastic/elasticsearch/pull/122458) Analysis: -* Analyze API to return 400 for wrong custom analyzer [#121568](https://github.com/elastic/elasticsearch/pull/121568) (issue: {es-issue}121443[#121443]) - -Authentication: -* Improve jwt logging on failed auth [#122247](https://github.com/elastic/elasticsearch/pull/122247) - -CAT APIs: -* Fix cat_component_templates documentation [#120487](https://github.com/elastic/elasticsearch/pull/120487) - -CRUD: -* Preserve thread context when waiting for segment generation in RTG [#114623](https://github.com/elastic/elasticsearch/pull/114623) -* Preserve thread context when waiting for segment generation in RTG [#117148](https://github.com/elastic/elasticsearch/pull/117148) -* Reduce license checks in `LicensedWriteLoadForecaster` [#123346](https://github.com/elastic/elasticsearch/pull/123346) (issue: {es-issue}123247[#123247]) +* Adjust exception thrown when unable to load hunspell dict [#123743](https://github.com/elastic/elasticsearch/pull/123743) Data streams: -* Add `_metric_names_hash` field to OTel metric mappings [#120952](https://github.com/elastic/elasticsearch/pull/120952) -* Avoid updating settings version in `MetadataMigrateToDataStreamService` when settings have not changed [#118704](https://github.com/elastic/elasticsearch/pull/118704) -* Block-writes cannot be added after read-only [#119007](https://github.com/elastic/elasticsearch/pull/119007) (issue: {es-issue}119002[#119002]) -* Ensure removal of index blocks does not leave key with null value [#122246](https://github.com/elastic/elasticsearch/pull/122246) -* Match dot prefix of migrated DS backing index with the source index [#120042](https://github.com/elastic/elasticsearch/pull/120042) -* Refresh source index before reindexing data stream index [#120752](https://github.com/elastic/elasticsearch/pull/120752) (issue: {es-issue}120314[#120314]) * Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover [#122905](https://github.com/elastic/elasticsearch/pull/122905) -* `ReindexDataStreamIndex` bug in assertion caused by reference equality [#121325](https://github.com/elastic/elasticsearch/pull/121325) - -Downsampling: -* Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` [#121727](https://github.com/elastic/elasticsearch/pull/121727) (issues: {es-issue}119696[#119696], {es-issue}96076[#96076]) - -EQL: -* Fix JOIN command validation (not supported) [#122011](https://github.com/elastic/elasticsearch/pull/122011) ES|QL: * Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: {es-issue}122413[#122413]) -* Allow the data type of `null` in filters [#118324](https://github.com/elastic/elasticsearch/pull/118324) (issue: {es-issue}116351[#116351]) * Avoid over collecting in Limit or Lucene Operator [#123296](https://github.com/elastic/elasticsearch/pull/123296) -* Correct line and column numbers of missing named parameters [#120852](https://github.com/elastic/elasticsearch/pull/120852) -* Drop null columns in text formats [#117643](https://github.com/elastic/elasticsearch/pull/117643) (issue: {es-issue}116848[#116848]) * ESQL: Fix inconsistent results in using scaled_float field [#122586](https://github.com/elastic/elasticsearch/pull/122586) (issue: {es-issue}122547[#122547]) -* Fix ENRICH validation for use of wildcards [#121911](https://github.com/elastic/elasticsearch/pull/121911) -* Fix ROUND() with unsigned longs throwing in some edge cases [#119536](https://github.com/elastic/elasticsearch/pull/119536) -* Fix TDigestState.read CB leaks [#114303](https://github.com/elastic/elasticsearch/pull/114303) (issue: {es-issue}114194[#114194]) -* Fix TopN row size estimate [#119476](https://github.com/elastic/elasticsearch/pull/119476) (issue: {es-issue}106956[#106956]) -* Fix `AbstractShapeGeometryFieldMapperTests` [#119265](https://github.com/elastic/elasticsearch/pull/119265) (issue: {es-issue}119201[#119201]) -* Fix a bug in TOP [#121552](https://github.com/elastic/elasticsearch/pull/121552) -* Fix async stop sometimes not properly collecting result [#121843](https://github.com/elastic/elasticsearch/pull/121843) (issue: {es-issue}121249[#121249]) -* Fix attribute set equals [#118823](https://github.com/elastic/elasticsearch/pull/118823) -* Fix double lookup failure on ESQL [#115616](https://github.com/elastic/elasticsearch/pull/115616) (issue: {es-issue}111398[#111398]) +* ESQL: Remove estimated row size assertion [#122762](https://github.com/elastic/elasticsearch/pull/122762) (issue: {es-issue}121535[#121535]) * Fix early termination in `LuceneSourceOperator` [#123197](https://github.com/elastic/elasticsearch/pull/123197) * Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: {es-issue}123430[#123430]) * Fix functions emitting warnings with no source [#122821](https://github.com/elastic/elasticsearch/pull/122821) (issue: {es-issue}122588[#122588]) -* Fix listener leak in exchange service [#122417](https://github.com/elastic/elasticsearch/pull/122417) (issue: {es-issue}122271[#122271]) -* Fix queries with document level security on lookup indexes [#120617](https://github.com/elastic/elasticsearch/pull/120617) (issue: {es-issue}120509[#120509]) -* Fix writing for LOOKUP status [#119296](https://github.com/elastic/elasticsearch/pull/119296) (issue: {es-issue}119086[#119086]) * Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: {es-issue}121890[#121890]) -* Limit memory usage of `fold` [#118602](https://github.com/elastic/elasticsearch/pull/118602) -* Limit size of query [#117898](https://github.com/elastic/elasticsearch/pull/117898) * Reduce iteration complexity for plan traversal [#123427](https://github.com/elastic/elasticsearch/pull/123427) * Remove duplicated nested commands [#123085](https://github.com/elastic/elasticsearch/pull/123085) -* Remove redundant sorts from execution plan [#121156](https://github.com/elastic/elasticsearch/pull/121156) -* Revert unwanted ES|QL lexer changes from PR #120354 [#120538](https://github.com/elastic/elasticsearch/pull/120538) * Revive inlinestats [#122257](https://github.com/elastic/elasticsearch/pull/122257) -* Speed up VALUES for many buckets [#123073](https://github.com/elastic/elasticsearch/pull/123073) +* Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: {es-issue}123967[#123967]) Engine: * Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: {es-issue}122974[#122974]) -Health: -* Do not recommend increasing `max_shards_per_node` [#120458](https://github.com/elastic/elasticsearch/pull/120458) - Indices APIs: -* Add `?master_timeout` to `POST /_ilm/migrate_to_data_tiers` [#120883](https://github.com/elastic/elasticsearch/pull/120883) -* Fix broken yaml test `30_create_from` [#120662](https://github.com/elastic/elasticsearch/pull/120662) -* Include hidden indices in `DeprecationInfoAction` [#118035](https://github.com/elastic/elasticsearch/pull/118035) (issue: {es-issue}118020[#118020]) * Updates the deprecation info API to not warn about system indices and data streams [#122951](https://github.com/elastic/elasticsearch/pull/122951) -Inference: -* [Inference API] Put back legacy EIS URL setting [#121207](https://github.com/elastic/elasticsearch/pull/121207) - Infra/Core: -* Epoch Millis Rounding Down and Not Up 2 [#118353](https://github.com/elastic/elasticsearch/pull/118353) * Include data streams when converting an existing resource to a system resource [#121392](https://github.com/elastic/elasticsearch/pull/121392) * Reduce Data Loss in System Indices Migration [#121327](https://github.com/elastic/elasticsearch/pull/121327) * System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) -* Wrap jackson exception on malformed json string [#114445](https://github.com/elastic/elasticsearch/pull/114445) (issue: {es-issue}114142[#114142]) - -Infra/Logging: -* Move `SlowLogFieldProvider` instantiation to node construction [#117949](https://github.com/elastic/elasticsearch/pull/117949) - -Infra/Metrics: -* Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]` [#114177](https://github.com/elastic/elasticsearch/pull/114177) - -Infra/Plugins: -* Remove unnecessary entitlement [#120959](https://github.com/elastic/elasticsearch/pull/120959) -* Restrict agent entitlements to the system classloader unnamed module [#120546](https://github.com/elastic/elasticsearch/pull/120546) - -Infra/REST API: -* Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null. [#113413](https://github.com/elastic/elasticsearch/pull/113413) (issue: {es-issue}113413[#113413]) - -Infra/Scripting: -* Register mustache size limit setting [#119291](https://github.com/elastic/elasticsearch/pull/119291) - -Infra/Settings: -* Don't allow secure settings in YML config (109115) [#115779](https://github.com/elastic/elasticsearch/pull/115779) (issue: {es-issue}109115[#109115]) - -Ingest: -* Fix `ArrayIndexOutOfBoundsException` in `ShardBulkInferenceActionFilter` [#122538](https://github.com/elastic/elasticsearch/pull/122538) Ingest Node: -* Add warning headers for ingest pipelines containing special characters [#114837](https://github.com/elastic/elasticsearch/pull/114837) (issue: {es-issue}104411[#104411]) -* Canonicalize processor names and types in `IngestStats` [#122610](https://github.com/elastic/elasticsearch/pull/122610) -* Deduplicate `IngestStats` and `IngestStats.Stats` identity records when deserializing [#122496](https://github.com/elastic/elasticsearch/pull/122496) -* Fix geoip databases index access after system feature migration [#121196](https://github.com/elastic/elasticsearch/pull/121196) * Fix geoip databases index access after system feature migration (again) [#122938](https://github.com/elastic/elasticsearch/pull/122938) -* Fix redact processor arraycopy bug [#122640](https://github.com/elastic/elasticsearch/pull/122640) -* Register `IngestGeoIpMetadata` as a NamedXContent [#123079](https://github.com/elastic/elasticsearch/pull/123079) -* Use ordered maps for `PipelineConfiguration` xcontent deserialization [#123403](https://github.com/elastic/elasticsearch/pull/123403) * apm-data: Use representative count as event.success_count if available [#119995](https://github.com/elastic/elasticsearch/pull/119995) -Logs: -* Always check if index mode is logsdb [#116922](https://github.com/elastic/elasticsearch/pull/116922) -* Fix issues that prevents using search only snapshots for indices that use index sorting. This is includes Logsdb and time series indices. [#122199](https://github.com/elastic/elasticsearch/pull/122199) -* Use min node version to guard injecting settings in logs provider [#123005](https://github.com/elastic/elasticsearch/pull/123005) (issue: {es-issue}122950[#122950]) - Machine Learning: * Add `ElasticInferenceServiceCompletionServiceSettings` [#123155](https://github.com/elastic/elasticsearch/pull/123155) * Add enterprise license check to inference action for semantic text fields [#122293](https://github.com/elastic/elasticsearch/pull/122293) -* Change format for Unified Chat [#121396](https://github.com/elastic/elasticsearch/pull/121396) -* Fix get all inference endponts not returning multiple endpoints sharing model deployment [#121821](https://github.com/elastic/elasticsearch/pull/121821) * Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) -* Fixing bedrock event executor terminated cache issue [#118177](https://github.com/elastic/elasticsearch/pull/118177) (issue: {es-issue}117916[#117916]) -* Fixing bug setting index when parsing Google Vertex AI results [#117287](https://github.com/elastic/elasticsearch/pull/117287) +* Retry on streaming errors [#123076](https://github.com/elastic/elasticsearch/pull/123076) * Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) * Updates to allow using Cohere binary embedding response in semantic search queries [#121827](https://github.com/elastic/elasticsearch/pull/121827) -* Updating Inference Update API documentation to have the correct PUT method [#121048](https://github.com/elastic/elasticsearch/pull/121048) -* Wait for up to 2 seconds for yellow status before starting search [#115938](https://github.com/elastic/elasticsearch/pull/115938) (issues: {es-issue}107777[#107777], {es-issue}105955[#105955], {es-issue}107815[#107815], {es-issue}112191[#112191]) -* [Inference API] Fix unique ID message for inference ID matches trained model ID [#119543](https://github.com/elastic/elasticsearch/pull/119543) (issue: {es-issue}111312[#111312]) - -Mapping: -* Enable New Semantic Text Format Only On Newly Created Indices [#121556](https://github.com/elastic/elasticsearch/pull/121556) -* Fix propagation of dynamic mapping parameter when applying `copy_to` [#121109](https://github.com/elastic/elasticsearch/pull/121109) (issue: {es-issue}113049[#113049]) -* Fix realtime get of nested fields with synthetic source [#119575](https://github.com/elastic/elasticsearch/pull/119575) (issue: {es-issue}119553[#119553]) -* Fix synthetic source bug that would mishandle nested `dense_vector` fields [#122425](https://github.com/elastic/elasticsearch/pull/122425) -* Merge field mappers when updating mappings with [subobjects:false] [#120370](https://github.com/elastic/elasticsearch/pull/120370) (issue: {es-issue}120216[#120216]) -* Tweak `copy_to` handling in synthetic `_source` to account for nested objects [#120974](https://github.com/elastic/elasticsearch/pull/120974) (issue: {es-issue}120831[#120831]) -* fix stale data in synthetic source for string stored field [#123105](https://github.com/elastic/elasticsearch/pull/123105) (issue: {es-issue}123110[#123110]) - -Network: -* Remove ChunkedToXContentBuilder [#119310](https://github.com/elastic/elasticsearch/pull/119310) (issue: {es-issue}118647[#118647]) Search: -* Catch and handle disconnect exceptions in search [#115836](https://github.com/elastic/elasticsearch/pull/115836) * Fix handling of auto expand replicas for stateless indices [#122365](https://github.com/elastic/elasticsearch/pull/122365) -* Fix leak in `DfsQueryPhase` and introduce search disconnect stress test [#116060](https://github.com/elastic/elasticsearch/pull/116060) (issue: {es-issue}115056[#115056]) -* Fix/QueryBuilderBWCIT_muted_test [#117831](https://github.com/elastic/elasticsearch/pull/117831) * Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: {es-issue}122186[#122186]) -* In this pr, a 400 error is returned when _source / _seq_no / _feature / _nested_path / _field_names is requested, rather a 5xx [#117229](https://github.com/elastic/elasticsearch/pull/117229) -* Inconsistency in the _analyzer api when the index is not included [#115930](https://github.com/elastic/elasticsearch/pull/115930) -* Remove duplicate code in ESIntegTestCase [#120799](https://github.com/elastic/elasticsearch/pull/120799) -* SearchStatesIt failures reported by CI [#117618](https://github.com/elastic/elasticsearch/pull/117618) (issues: {es-issue}116617[#116617], {es-issue}116618[#116618]) -* Skip fetching _inference_fields field in legacy semantic_text format [#121720](https://github.com/elastic/elasticsearch/pull/121720) -* Test/107515 restore template with match only text mapper it fail [#120392](https://github.com/elastic/elasticsearch/pull/120392) (issue: {es-issue}107515[#107515]) -* Updated Date Range to Follow Documentation When Assuming Missing Values [#112258](https://github.com/elastic/elasticsearch/pull/112258) (issue: {es-issue}111484[#111484]) -* `CrossClusterIT` `testCancel` failure [#117750](https://github.com/elastic/elasticsearch/pull/117750) (issue: {es-issue}108061[#108061]) -* `SearchServiceTests.testParseSourceValidation` failure [#117963](https://github.com/elastic/elasticsearch/pull/117963) Snapshot/Restore: -* Add undeclared Azure settings, modify test to exercise them [#118634](https://github.com/elastic/elasticsearch/pull/118634) * Fork post-snapshot-delete cleanup off master thread [#122731](https://github.com/elastic/elasticsearch/pull/122731) * Limit number of suppressed S3 deletion errors [#123630](https://github.com/elastic/elasticsearch/pull/123630) (issue: {es-issue}123354[#123354]) -* Retry throttled snapshot deletions [#113237](https://github.com/elastic/elasticsearch/pull/113237) -* Use the system index descriptor in the snapshot blob cache cleanup task [#120937](https://github.com/elastic/elasticsearch/pull/120937) (issue: {es-issue}120518[#120518]) - -Stats: -* Fixing serialization of `ScriptStats` `cache_evictions_history` [#123384](https://github.com/elastic/elasticsearch/pull/123384) - -Store: -* Do not capture `ClusterChangedEvent` in `IndicesStore` call to #onClusterStateShardsClosed [#120193](https://github.com/elastic/elasticsearch/pull/120193) Suggesters: * Return an empty suggestion when suggest phase times out [#122575](https://github.com/elastic/elasticsearch/pull/122575) (issue: {es-issue}122548[#122548]) @@ -649,31 +180,10 @@ Transform: * If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias [#122074](https://github.com/elastic/elasticsearch/pull/122074) (issue: {es-issue}121913[#121913]) Vector Search: -* Apply default k for knn query eagerly [#118774](https://github.com/elastic/elasticsearch/pull/118774) -* Fix `bbq_hnsw` merge file cleanup on random IO exceptions [#119691](https://github.com/elastic/elasticsearch/pull/119691) (issue: {es-issue}119392[#119392]) * Knn vector rescoring to sort score docs [#122653](https://github.com/elastic/elasticsearch/pull/122653) (issue: {es-issue}119711[#119711]) -Watcher: -* Watcher history index has too many indexed fields - [#117701](https://github.com/elastic/elasticsearch/pull/117701) (issue: {es-issue}71479[#71479]) - ### Upgrades [elasticsearch-910-upgrade] -Authentication: -* Bump json-smart and oauth2-oidc-sdk [#122737](https://github.com/elastic/elasticsearch/pull/122737) - -Infra/Core: -* Bump major version for feature migration system indices [#117243](https://github.com/elastic/elasticsearch/pull/117243) -* Update ASM 9.7 -> 9.7.1 to support JDK 24 [#118094](https://github.com/elastic/elasticsearch/pull/118094) - -Machine Learning: -* Automatically rollover legacy .ml-anomalies indices [#120913](https://github.com/elastic/elasticsearch/pull/120913) -* Automatically rollover legacy ml indices [#120405](https://github.com/elastic/elasticsearch/pull/120405) -* Change the auditor to write via an alias [#120064](https://github.com/elastic/elasticsearch/pull/120064) - -Search: -* Upgrade to Lucene 10 [#114741](https://github.com/elastic/elasticsearch/pull/114741) -* Upgrade to Lucene 10.1.0 [#119308](https://github.com/elastic/elasticsearch/pull/119308) - Snapshot/Restore: * Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) From 2b1a1406e72bbfdb4f4d3c429e39b1b508f9ae8c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 6 Mar 2025 20:39:10 +0000 Subject: [PATCH 04/34] [CI] Auto commit changes from spotless --- .../gradle/internal/release/ReleaseToolsPlugin.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index ea222442e9445..8acd074f7e843 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -92,9 +92,7 @@ public void apply(Project project) { task.setMigrationIndexFile(projectDirectory.file("docs/reference/migration/index.asciidoc")); task.setDeprecationsTemplate(projectDirectory.file(RESOURCES + "templates/deprecations.md")); - task.setDeprecationsFile( - projectDirectory.file("docs/release-notes/deprecations.md") - ); + task.setDeprecationsFile(projectDirectory.file("docs/release-notes/deprecations.md")); task.dependsOn(validateChangelogsTask); }; From 364a306307ae3d81790eda638580ead1939c7f45 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 11 Mar 2025 16:16:50 -0400 Subject: [PATCH 05/34] Organizational changes based on direction from docs team, cleanup and simplify some stuff --- .../release/BreakingChangesGenerator.java | 85 ---------------- .../release/GenerateReleaseNotesTask.java | 81 ++------------- .../release/MigrationIndexGenerator.java | 51 ---------- .../release/ReleaseNotesGenerator.java | 11 ++- .../internal/release/ReleaseToolsPlugin.java | 9 +- .../templates/breaking-changes.asciidoc | 98 ------------------- .../templates/{release-notes.md => index.md} | 5 +- .../templates/release-notes-index.asciidoc | 12 --- .../templates/release-notes.asciidoc | 45 --------- docs/release-notes/deprecations.md | 2 - docs/release-notes/index.md | 57 +++++++---- 11 files changed, 63 insertions(+), 393 deletions(-) delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java delete mode 100644 build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc rename build-tools-internal/src/main/resources/templates/{release-notes.md => index.md} (91%) delete mode 100644 build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc delete mode 100644 build-tools-internal/src/main/resources/templates/release-notes.asciidoc diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java deleted file mode 100644 index 849e05742f2b8..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import com.google.common.annotations.VisibleForTesting; - -import org.elasticsearch.gradle.VersionProperties; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.TreeMap; - -import static java.util.Comparator.comparing; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toList; - -/** - * Generates the page that contains breaking changes deprecations for a minor release series. - */ -public class BreakingChangesGenerator { - - static void update(File migrationTemplateFile, File migrationOutputFile, List entries) throws IOException { - try (FileWriter output = new FileWriter(migrationOutputFile)) { - output.write( - generateMigrationFile( - QualifiedVersion.of(VersionProperties.getElasticsearch()), - Files.readString(migrationTemplateFile.toPath()), - entries - ) - ); - } - } - - @VisibleForTesting - static String generateMigrationFile(QualifiedVersion version, String template, List entries) throws IOException { - final Map>> deprecationsByNotabilityByArea = entries.stream() - .map(ChangelogEntry::getDeprecation) - .filter(Objects::nonNull) - .sorted(comparing(ChangelogEntry.Deprecation::getTitle)) - .collect( - groupingBy( - ChangelogEntry.Deprecation::isNotable, - TreeMap::new, - groupingBy(ChangelogEntry.Deprecation::getArea, TreeMap::new, toList()) - ) - ); - - final Map>> breakingByNotabilityByArea = entries.stream() - .map(ChangelogEntry::getBreaking) - .filter(Objects::nonNull) - .sorted(comparing(ChangelogEntry.Breaking::getTitle)) - .collect( - groupingBy( - ChangelogEntry.Breaking::isNotable, - TreeMap::new, - groupingBy(ChangelogEntry.Breaking::getArea, TreeMap::new, toList()) - ) - ); - - final Map bindings = new HashMap<>(); - bindings.put("breakingByNotabilityByArea", breakingByNotabilityByArea); - bindings.put("deprecationsByNotabilityByArea", deprecationsByNotabilityByArea); - bindings.put("isElasticsearchSnapshot", version.isSnapshot()); - bindings.put("majorDotMinor", version.major() + "." + version.minor()); - bindings.put("majorDotMinorDotRevision", version.major() + "." + version.minor() + "." + version.revision()); - bindings.put("majorMinor", String.valueOf(version.major()) + version.minor()); - bindings.put("nextMajor", (version.major() + 1) + ".0"); - bindings.put("version", version); - - return TemplateUtils.render(template, bindings); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index 70cb214fc8c12..79e61d833a858 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -53,19 +53,15 @@ public class GenerateReleaseNotesTask extends DefaultTask { private final ConfigurableFileCollection changelogs; - private final RegularFileProperty releaseNotesIndexTemplate; private final RegularFileProperty releaseNotesTemplate; private final RegularFileProperty releaseHighlightsTemplate; private final RegularFileProperty breakingChangesTemplate; private final RegularFileProperty deprecationsTemplate; - private final RegularFileProperty migrationIndexTemplate; - private final RegularFileProperty releaseNotesIndexFile; private final RegularFileProperty releaseNotesFile; private final RegularFileProperty releaseHighlightsFile; - private final RegularFileProperty breakingChangesMigrationFile; + private final RegularFileProperty breakingChangesFile; private final RegularFileProperty deprecationsFile; - private final RegularFileProperty migrationIndexFile; private final GitWrapper gitWrapper; @@ -73,19 +69,15 @@ public class GenerateReleaseNotesTask extends DefaultTask { public GenerateReleaseNotesTask(ObjectFactory objectFactory, ExecOperations execOperations) { changelogs = objectFactory.fileCollection(); - releaseNotesIndexTemplate = objectFactory.fileProperty(); releaseNotesTemplate = objectFactory.fileProperty(); releaseHighlightsTemplate = objectFactory.fileProperty(); breakingChangesTemplate = objectFactory.fileProperty(); deprecationsTemplate = objectFactory.fileProperty(); - migrationIndexTemplate = objectFactory.fileProperty(); - releaseNotesIndexFile = objectFactory.fileProperty(); releaseNotesFile = objectFactory.fileProperty(); releaseHighlightsFile = objectFactory.fileProperty(); - breakingChangesMigrationFile = objectFactory.fileProperty(); + breakingChangesFile = objectFactory.fileProperty(); deprecationsFile = objectFactory.fileProperty(); - migrationIndexFile = objectFactory.fileProperty(); gitWrapper = new GitWrapper(execOperations); } @@ -117,13 +109,6 @@ public void executeTask() throws IOException { final Set versions = getVersions(gitWrapper, currentVersion); - LOGGER.info("Updating release notes index..."); - ReleaseNotesIndexGenerator.update( - versions, - this.releaseNotesIndexTemplate.get().getAsFile(), - this.releaseNotesIndexFile.get().getAsFile() - ); - LOGGER.info("Generating release notes..."); final QualifiedVersion qualifiedVersion = QualifiedVersion.of(currentVersion); ReleaseNotesGenerator.update( @@ -133,19 +118,12 @@ public void executeTask() throws IOException { changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) ); - // Only update breaking changes and migration guide for new minors + // Only update breaking changes and deprecations for new minors if (qualifiedVersion.revision() == 0) { - LOGGER.info("Generating release highlights..."); - ReleaseHighlightsGenerator.update( - this.releaseHighlightsTemplate.get().getAsFile(), - this.releaseHighlightsFile.get().getAsFile(), - entries - ); - LOGGER.info("Generating breaking changes / deprecations notes..."); ReleaseNotesGenerator.update( this.breakingChangesTemplate.get().getAsFile(), - this.breakingChangesMigrationFile.get().getAsFile(), + this.breakingChangesFile.get().getAsFile(), qualifiedVersion, changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) ); @@ -156,13 +134,6 @@ public void executeTask() throws IOException { qualifiedVersion, changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) ); - - LOGGER.info("Updating migration/index..."); - MigrationIndexGenerator.update( - getMinorVersions(versions), - this.migrationIndexTemplate.get().getAsFile(), - this.migrationIndexFile.get().getAsFile() - ); } } @@ -329,15 +300,6 @@ public void setChangelogs(FileCollection files) { this.changelogs.setFrom(files); } - @InputFile - public RegularFileProperty getReleaseNotesIndexTemplate() { - return releaseNotesIndexTemplate; - } - - public void setReleaseNotesIndexTemplate(RegularFile file) { - this.releaseNotesIndexTemplate.set(file); - } - @InputFile public RegularFileProperty getReleaseNotesTemplate() { return releaseNotesTemplate; @@ -374,24 +336,6 @@ public void setDeprecationsTemplate(RegularFile file) { this.deprecationsTemplate.set(file); } - @InputFile - public RegularFileProperty getMigrationIndexTemplate() { - return migrationIndexTemplate; - } - - public void setMigrationIndexTemplate(RegularFile file) { - this.migrationIndexTemplate.set(file); - } - - @OutputFile - public RegularFileProperty getReleaseNotesIndexFile() { - return releaseNotesIndexFile; - } - - public void setReleaseNotesIndexFile(RegularFile file) { - this.releaseNotesIndexFile.set(file); - } - @OutputFile public RegularFileProperty getReleaseNotesFile() { return releaseNotesFile; @@ -411,12 +355,12 @@ public void setReleaseHighlightsFile(RegularFile file) { } @OutputFile - public RegularFileProperty getBreakingChangesMigrationFile() { - return breakingChangesMigrationFile; + public RegularFileProperty getBreakingChangesFile() { + return breakingChangesFile; } - public void setBreakingChangesMigrationFile(RegularFile file) { - this.breakingChangesMigrationFile.set(file); + public void setBreakingChangesFile(RegularFile file) { + this.breakingChangesFile.set(file); } @OutputFile @@ -427,13 +371,4 @@ public RegularFileProperty getDeprecationsFile() { public void setDeprecationsFile(RegularFile file) { this.deprecationsFile.set(file); } - - @OutputFile - public RegularFileProperty getMigrationIndexFile() { - return migrationIndexFile; - } - - public void setMigrationIndexFile(RegularFile file) { - this.migrationIndexFile.set(file); - } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java deleted file mode 100644 index 668da0da6d0d0..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import com.google.common.annotations.VisibleForTesting; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; - -import static java.util.Comparator.reverseOrder; - -/** - * This class ensures that the migrate/index page has the appropriate anchors and include directives - * for the current repository version. - */ -public class MigrationIndexGenerator { - - static void update(Set versions, File indexTemplate, File indexFile) throws IOException { - try (FileWriter indexFileWriter = new FileWriter(indexFile)) { - indexFileWriter.write(generateFile(versions, Files.readString(indexTemplate.toPath()))); - } - } - - @VisibleForTesting - static String generateFile(Set versionsSet, String template) throws IOException { - final Set versions = new TreeSet<>(reverseOrder()); - versions.addAll(versionsSet); - final List includeVersions = versions.stream().map(MinorVersion::underscore).collect(Collectors.toList()); - - final Map bindings = new HashMap<>(); - bindings.put("versions", versions); - bindings.put("includeVersions", includeVersions); - - return TemplateUtils.render(template, bindings); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java index 4f615fc9167e8..3a14bdb74e04c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java @@ -49,6 +49,11 @@ public class ReleaseNotesGenerator { TYPE_LABELS.put("upgrade", "Upgrades"); } + /** + * These are the types of changes that are considered "Features and Enhancements" in the release notes. + */ + private static final List FEATURE_ENHANCEMENT_TYPES = List.of("feature", "new-aggregation", "enhancement", "upgrade"); + static void update(File templateFile, File outputFile, QualifiedVersion version, Set changelogs) throws IOException { final String templateString = Files.readString(templateFile.toPath()); @@ -71,12 +76,16 @@ static String generateFile(String template, QualifiedVersion version, Set${majorDotMinor} -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} ${majorDotMinor}. - -See also <> and <>. -<% if (isElasticsearchSnapshot) { %> -coming::[${majorDotMinorDotRevision}] -<% } %> - -[discrete] -[[breaking-changes-${majorDotMinor}]] -=== Breaking changes -<% if (breakingByNotabilityByArea.isEmpty()) { %> -There are no breaking changes in {es} ${majorDotMinor}. -<% } else { %> -The following changes in {es} ${majorDotMinor} might affect your applications -and prevent them from operating normally. -Before upgrading to ${majorDotMinor}, review these changes and take the described steps -to mitigate the impact. -<% - if (breakingByNotabilityByArea.getOrDefault(true, []).isEmpty()) { %> - -There are no notable breaking changes in {es} ${majorDotMinor}. -But there are some less critical breaking changes. -<% } - [true, false].each { isNotable -> - def breakingByArea = breakingByNotabilityByArea.getOrDefault(isNotable, []) - if (breakingByArea.isEmpty() == false) { - breakingByArea.eachWithIndex { area, breakingChanges, i -> - print "\n[discrete]\n" - print "[[breaking_${majorMinor}_${ area.toLowerCase().replaceAll("[^a-z0-9]+", "_") }_changes]]\n" - print "==== ${area} changes\n" - - for (breaking in breakingChanges) { %> -[[${ breaking.anchor }]] -.${breaking.title} -[%collapsible] -==== -*Details* + -${breaking.details.trim()} - -*Impact* + -${breaking.impact.trim()} -==== -<% - } - } - - } - } -} - -if (deprecationsByNotabilityByArea.isEmpty() == false) { %> - -[discrete] -[[deprecated-${majorDotMinor}]] -=== Deprecations - -The following functionality has been deprecated in {es} ${majorDotMinor} -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to ${majorDotMinor}. - -To find out if you are using any deprecated functionality, -enable <>. -<% - [true, false].each { isNotable -> - def deprecationsByArea = deprecationsByNotabilityByArea.getOrDefault(isNotable, []) - if (deprecationsByArea.isEmpty() == false) { - deprecationsByArea.eachWithIndex { area, deprecations, i -> - print "\n[discrete]\n" - print "[[deprecations_${majorMinor}_${ area.toLowerCase().replaceAll("[^a-z0-9]+", "_") }]]\n" - print "==== ${area} deprecations\n" - - for (deprecation in deprecations) { %> -[[${ deprecation.anchor }]] -.${deprecation.title} -[%collapsible] -==== -*Details* + -${deprecation.details.trim()} - -*Impact* + -${deprecation.impact.trim()} -==== -<% - } - } - - } - } -} %> diff --git a/build-tools-internal/src/main/resources/templates/release-notes.md b/build-tools-internal/src/main/resources/templates/index.md similarity index 91% rename from build-tools-internal/src/main/resources/templates/release-notes.md rename to build-tools-internal/src/main/resources/templates/index.md index 1ca6cc76b4252..8ef24eee590c0 100644 --- a/build-tools-internal/src/main/resources/templates/release-notes.md +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -27,7 +27,10 @@ To check for security updates, go to [Security announcements for the Elastic sta ## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-release-notes] **Release date:** April 01, 2025 <% -for (changeType in changelogsByTypeByArea.keySet()) { +for (changeType in ['features-enhancements', 'fixes', 'regression']) { + if (changelogsByTypeByArea[changeType] == null || changelogsByTypeByArea[changeType].empty) { + continue; + } %> ### ${ TYPE_LABELS.getOrDefault(changeType, 'No mapping for TYPE_LABELS[' + changeType + ']') } [elasticsearch-${versionWithoutSeparator}-${changeType}] <% for (team in changelogsByTypeByArea[changeType].keySet()) { diff --git a/build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc b/build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc deleted file mode 100644 index ba30aea3bf14e..0000000000000 --- a/build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc +++ /dev/null @@ -1,12 +0,0 @@ -[[es-release-notes]] -= Release notes - -[partintro] --- - -This section summarizes the changes in each release. - -<% versions.each { print "* <>\n" } %> --- - -<% includeVersions.each { print "include::release-notes/${ it }.asciidoc[]\n" } %> diff --git a/build-tools-internal/src/main/resources/templates/release-notes.asciidoc b/build-tools-internal/src/main/resources/templates/release-notes.asciidoc deleted file mode 100644 index 096608435333a..0000000000000 --- a/build-tools-internal/src/main/resources/templates/release-notes.asciidoc +++ /dev/null @@ -1,45 +0,0 @@ -<% -def unqualifiedVersion = version.withoutQualifier() -%>[[release-notes-$unqualifiedVersion]] -== {es} version ${unqualifiedVersion} -<% if (version.isSnapshot()) { %> -coming[$unqualifiedVersion] -<% } %> -Also see <>. -<% if (changelogsByTypeByArea["security"] != null) { %> -[discrete] -[[security-updates-${unqualifiedVersion}]] -=== Security updates - -<% for (change in changelogsByTypeByArea.remove("security").remove("_all_")) { - print "* ${change.summary}\n" -} -} -if (changelogsByTypeByArea["known-issue"] != null) { %> -[discrete] -[[known-issues-${unqualifiedVersion}]] -=== Known issues - -<% for (change in changelogsByTypeByArea.remove("known-issue").remove("_all_")) { - print "* ${change.summary}\n" -} -} -for (changeType in changelogsByTypeByArea.keySet()) { %> -[[${ changeType }-${ unqualifiedVersion }]] -[float] -=== ${ TYPE_LABELS.getOrDefault(changeType, 'No mapping for TYPE_LABELS[' + changeType + ']') } -<% for (team in changelogsByTypeByArea[changeType].keySet()) { - print "\n${team}::\n"; - - for (change in changelogsByTypeByArea[changeType][team]) { - print "* ${change.summary} {es-pull}${change.pr}[#${change.pr}]" - if (change.issues != null && change.issues.empty == false) { - print change.issues.size() == 1 ? " (issue: " : " (issues: " - print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") - print ")" - } - print "\n" - } -} -} -print "\n\n" diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index 2e2269249ad41..390d605135b9d 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -20,5 +20,3 @@ To give you insight into what deprecated features you’re using, {{es}}: **Release date:** April 01, 2025 No deprecations in this version. - - diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 5294609c89299..c53b866871f5e 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -27,19 +27,6 @@ To check for security updates, go to [Security announcements for the Elastic sta ## 9.1.0 [elasticsearch-910-release-notes] **Release date:** April 01, 2025 -### Breaking changes [elasticsearch-910-breaking] - -TLS: -* Drop `TLS_RSA` cipher support for JDK 24 [#123600](https://github.com/elastic/elasticsearch/pull/123600) - -### Deprecations [elasticsearch-910-deprecation] - -Machine Learning: -* Add deprecation warning for flush API [#121667](https://github.com/elastic/elasticsearch/pull/121667) (issue: {es-issue}121506[#121506]) - -Search: -* Deprecate Behavioral Analytics CRUD apis [#122960](https://github.com/elastic/elasticsearch/pull/122960) - ### Features and enhancements [elasticsearch-910-features-enhancements] Allocation: @@ -48,18 +35,27 @@ Allocation: Authorization: * Do not fetch reserved roles from native store when Get Role API is called [#121971](https://github.com/elastic/elasticsearch/pull/121971) +CRUD: +* Enhance memory accounting for document expansion and introduce max document size limit [#123543](https://github.com/elastic/elasticsearch/pull/123543) + Data streams: * Add index mode to get data stream API [#122486](https://github.com/elastic/elasticsearch/pull/122486) +* Retry ILM async action after reindexing data stream [#124149](https://github.com/elastic/elasticsearch/pull/124149) +* Set cause on create index request in create from action [#124363](https://github.com/elastic/elasticsearch/pull/124363) ES|QL: * Add initial grammar and changes for FORK [#121948](https://github.com/elastic/elasticsearch/pull/121948) +* Add initial grammar and planning for RRF (snapshot) [#123396](https://github.com/elastic/elasticsearch/pull/123396) * Allow partial results in ES|QL [#121942](https://github.com/elastic/elasticsearch/pull/121942) * Allow skip shards with `_tier` and `_index` in ES|QL [#123728](https://github.com/elastic/elasticsearch/pull/123728) +* Avoid `NamedWritable` in block serialization [#124394](https://github.com/elastic/elasticsearch/pull/124394) +* ES|QL - Add scoring for full text functions disjunctions [#121793](https://github.com/elastic/elasticsearch/pull/121793) * Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: {es-issue}122967[#122967]) * Implement runtime skip_unavailable=true [#121240](https://github.com/elastic/elasticsearch/pull/121240) * Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) * Introduce `allow_partial_results` setting in ES|QL [#122890](https://github.com/elastic/elasticsearch/pull/122890) * Introduce a pre-mapping logical plan processing step [#121260](https://github.com/elastic/elasticsearch/pull/121260) +* Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: {es-issue}123067[#123067]) * Render `aggregate_metric_double` [#122660](https://github.com/elastic/elasticsearch/pull/122660) * Retry ES|QL node requests on shard level failures [#120774](https://github.com/elastic/elasticsearch/pull/120774) * Support partial results in CCS in ES|QL [#122708](https://github.com/elastic/elasticsearch/pull/122708) @@ -70,11 +66,13 @@ Health: ILM+SLM: * Improve SLM Health Indicator to cover missing snapshot [#121370](https://github.com/elastic/elasticsearch/pull/121370) +* Run `TransportExplainLifecycleAction` on local node [#122885](https://github.com/elastic/elasticsearch/pull/122885) Inference: * [Inference API] Rename `model_id` prop to model in EIS sparse inference request body [#122272](https://github.com/elastic/elasticsearch/pull/122272) Infra/Core: +* Give Kibana user 'all' permissions for .entity_analytics.* indices [#123588](https://github.com/elastic/elasticsearch/pull/123588) * Improve size limiting string message [#122427](https://github.com/elastic/elasticsearch/pull/122427) Ingest Node: @@ -100,10 +98,20 @@ Mapping: * Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields [#124050](https://github.com/elastic/elasticsearch/pull/124050) * Use `FallbackSyntheticSourceBlockLoader` for number fields [#122280](https://github.com/elastic/elasticsearch/pull/122280) +Relevance: +* Skip semantic_text embedding generation when no content is provided. [#123763](https://github.com/elastic/elasticsearch/pull/123763) + Search: * Account for the `SearchHit` source in circuit breaker [#121920](https://github.com/elastic/elasticsearch/pull/121920) (issue: {es-issue}89656[#89656]) +* Added optional parameters to QSTR ES|QL function [#121787](https://github.com/elastic/elasticsearch/pull/121787) (issue: {es-issue}120933[#120933]) * Optionally allow text similarity reranking to fail [#121784](https://github.com/elastic/elasticsearch/pull/121784) +Security: +* Bump nimbus-jose-jwt to 10.0.2 [#124544](https://github.com/elastic/elasticsearch/pull/124544) + +Snapshot/Restore: +* Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) + Stats: * Run XPack usage actions on local node [#122933](https://github.com/elastic/elasticsearch/pull/122933) @@ -111,6 +119,7 @@ Store: * Abort pending deletion on `IndicesService` close [#123569](https://github.com/elastic/elasticsearch/pull/123569) Vector Search: +* Add bit vector support to semantic text [#123187](https://github.com/elastic/elasticsearch/pull/123187) * Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields [#122381](https://github.com/elastic/elasticsearch/pull/122381) (issue: {es-issue}117274[#117274]) Watcher: @@ -127,6 +136,9 @@ Analysis: Data streams: * Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover [#122905](https://github.com/elastic/elasticsearch/pull/122905) +Downsampling: +* Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. [#124451](https://github.com/elastic/elasticsearch/pull/124451) + ES|QL: * Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: {es-issue}122413[#122413]) * Avoid over collecting in Limit or Lucene Operator [#123296](https://github.com/elastic/elasticsearch/pull/123296) @@ -136,18 +148,22 @@ ES|QL: * Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: {es-issue}123430[#123430]) * Fix functions emitting warnings with no source [#122821](https://github.com/elastic/elasticsearch/pull/122821) (issue: {es-issue}122588[#122588]) * Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: {es-issue}121890[#121890]) +* Lazy collection copying during node transform [#124424](https://github.com/elastic/elasticsearch/pull/124424) * Reduce iteration complexity for plan traversal [#123427](https://github.com/elastic/elasticsearch/pull/123427) * Remove duplicated nested commands [#123085](https://github.com/elastic/elasticsearch/pull/123085) * Revive inlinestats [#122257](https://github.com/elastic/elasticsearch/pull/122257) +* Revive some more of inlinestats functionality [#123589](https://github.com/elastic/elasticsearch/pull/123589) * Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: {es-issue}123967[#123967]) Engine: * Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: {es-issue}122974[#122974]) Indices APIs: +* Avoid hoarding cluster state references during rollover [#124107](https://github.com/elastic/elasticsearch/pull/124107) (issue: {es-issue}123893[#123893]) * Updates the deprecation info API to not warn about system indices and data streams [#122951](https://github.com/elastic/elasticsearch/pull/122951) Infra/Core: +* Have create index return a bad request on poor formatting [#123761](https://github.com/elastic/elasticsearch/pull/123761) * Include data streams when converting an existing resource to a system resource [#121392](https://github.com/elastic/elasticsearch/pull/121392) * Reduce Data Loss in System Indices Migration [#121327](https://github.com/elastic/elasticsearch/pull/121327) * System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) @@ -159,12 +175,22 @@ Ingest Node: Machine Learning: * Add `ElasticInferenceServiceCompletionServiceSettings` [#123155](https://github.com/elastic/elasticsearch/pull/123155) * Add enterprise license check to inference action for semantic text fields [#122293](https://github.com/elastic/elasticsearch/pull/122293) +* Avoid potentially throwing calls to Task#getDescription in model download [#124527](https://github.com/elastic/elasticsearch/pull/124527) * Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) * Retry on streaming errors [#123076](https://github.com/elastic/elasticsearch/pull/123076) * Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) * Updates to allow using Cohere binary embedding response in semantic search queries [#121827](https://github.com/elastic/elasticsearch/pull/121827) +* [Inference API] Fix output stream ordering in `InferenceActionProxy` [#124225](https://github.com/elastic/elasticsearch/pull/124225) + +Mapping: +* Avoid serializing empty `_source` fields in mappings [#122606](https://github.com/elastic/elasticsearch/pull/122606) + +Ranking: +* Restore `TextSimilarityRankBuilder` XContent output [#124564](https://github.com/elastic/elasticsearch/pull/124564) Search: +* Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions [#123890](https://github.com/elastic/elasticsearch/pull/123890) +* Fix concurrency issue in `ScriptSortBuilder` [#123757](https://github.com/elastic/elasticsearch/pull/123757) * Fix handling of auto expand replicas for stateless indices [#122365](https://github.com/elastic/elasticsearch/pull/122365) * Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: {es-issue}122186[#122186]) @@ -182,9 +208,4 @@ Transform: Vector Search: * Knn vector rescoring to sort score docs [#122653](https://github.com/elastic/elasticsearch/pull/122653) (issue: {es-issue}119711[#119711]) -### Upgrades [elasticsearch-910-upgrade] - -Snapshot/Restore: -* Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) - From 5a0b964e5f66f6f9e49c8056a103858561fca8fd Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 12 Mar 2025 16:48:30 -0400 Subject: [PATCH 06/34] Update tests for release notes generation --- .../resources/templates/breaking-changes.md | 2 +- .../main/resources/templates/deprecations.md | 6 +- .../src/main/resources/templates/index.md | 2 +- .../release/BreakingChangesGeneratorTest.java | 130 ----------------- .../ReleaseNotesIndexGeneratorTest.java | 60 -------- ...neratorTest.generateMigrationFile.asciidoc | 134 ------------------ ...easeNotesGeneratorTest.breaking-changes.md | 22 +++ .../ReleaseNotesGeneratorTest.deprecations.md | 26 ++++ ...seNotesGeneratorTest.generateFile.asciidoc | 44 ------ .../ReleaseNotesGeneratorTest.index.md | 68 +++++++++ ...esIndexGeneratorTest.generateFile.asciidoc | 30 ---- docs/release-notes/breaking-changes.md | 2 +- docs/release-notes/deprecations.md | 8 +- docs/release-notes/index.md | 2 +- 14 files changed, 130 insertions(+), 406 deletions(-) delete mode 100644 build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java delete mode 100644 build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java delete mode 100644 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc create mode 100644 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md create mode 100644 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md delete mode 100644 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.generateFile.asciidoc create mode 100644 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md delete mode 100644 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.generateFile.asciidoc diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.md b/build-tools-internal/src/main/resources/templates/breaking-changes.md index 67bf5c0283831..d1bb4620ad6bc 100644 --- a/build-tools-internal/src/main/resources/templates/breaking-changes.md +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.md @@ -7,7 +7,7 @@ mapped_pages: # Elasticsearch breaking changes [elasticsearch-breaking-changes] Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. -To learn how to upgrade, check out . +To learn how to upgrade, check out . % ## Next version [elasticsearch-nextversion-breaking-changes] % **Release date:** Month day, year diff --git a/build-tools-internal/src/main/resources/templates/deprecations.md b/build-tools-internal/src/main/resources/templates/deprecations.md index b2b5e35f8d552..5f68745c65301 100644 --- a/build-tools-internal/src/main/resources/templates/deprecations.md +++ b/build-tools-internal/src/main/resources/templates/deprecations.md @@ -19,13 +19,13 @@ To give you insight into what deprecated features you’re using, {{es}}: ## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-deprecations] **Release date:** April 01, 2025 <% - if (!changelogsByTypeByArea['deprecations']) { + if (!changelogsByTypeByArea['deprecation']) { print "\nNo deprecations in this version.\n" } else { - for (team in (changelogsByTypeByArea['deprecations'] ?: [:]).keySet()) { + for (team in (changelogsByTypeByArea['deprecation'] ?: [:]).keySet()) { print "\n${team}:\n"; - for (change in changelogsByTypeByArea['deprecations'][team]) { + for (change in changelogsByTypeByArea['deprecation'][team]) { print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" if (change.issues != null && change.issues.empty == false) { print change.issues.size() == 1 ? " (issue: " : " (issues: " diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md index 8ef24eee590c0..bb6260c3b285b 100644 --- a/build-tools-internal/src/main/resources/templates/index.md +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -15,7 +15,7 @@ To check for security updates, go to [Security announcements for the Elastic sta % Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. -% ## version.next [felasticsearch-next-release-notes] +% ## version.next [elasticsearch-next-release-notes] % **Release date:** Month day, year % ### Features and enhancements [elasticsearch-next-features-enhancements] diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java deleted file mode 100644 index 5d51f3514cbc3..0000000000000 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import java.util.Objects; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -public class BreakingChangesGeneratorTest { - - /** - * Check that the breaking changes can be correctly generated. - */ - @Test - public void generateIndexFile_rendersCorrectMarkup() throws Exception { - // given: - final String template = getResource("/templates/breaking-changes.asciidoc"); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc" - ); - - final List entries = getEntries(); - - // when: - final String actualOutput = BreakingChangesGenerator.generateMigrationFile( - QualifiedVersion.of("8.4.0-SNAPSHOT"), - template, - entries - ); - - // then: - assertThat(actualOutput, equalTo(expectedOutput)); - } - - private List getEntries() { - ChangelogEntry entry1 = new ChangelogEntry(); - ChangelogEntry.Breaking breaking1 = new ChangelogEntry.Breaking(); - entry1.setBreaking(breaking1); - - breaking1.setNotable(true); - breaking1.setTitle("Breaking change number 1"); - breaking1.setArea("API"); - breaking1.setDetails("Breaking change details 1"); - breaking1.setImpact("Breaking change impact description 1"); - - ChangelogEntry entry2 = new ChangelogEntry(); - ChangelogEntry.Breaking breaking2 = new ChangelogEntry.Breaking(); - entry2.setBreaking(breaking2); - - breaking2.setNotable(true); - breaking2.setTitle("Breaking change number 2"); - breaking2.setArea("Cluster and node setting"); - breaking2.setDetails("Breaking change details 2"); - breaking2.setImpact("Breaking change impact description 2"); - - ChangelogEntry entry3 = new ChangelogEntry(); - ChangelogEntry.Breaking breaking3 = new ChangelogEntry.Breaking(); - entry3.setBreaking(breaking3); - - breaking3.setNotable(false); - breaking3.setTitle("Breaking change number 3"); - breaking3.setArea("Transform"); - breaking3.setDetails("Breaking change details 3"); - breaking3.setImpact("Breaking change impact description 3"); - - ChangelogEntry entry4 = new ChangelogEntry(); - ChangelogEntry.Breaking breaking4 = new ChangelogEntry.Breaking(); - entry4.setBreaking(breaking4); - - breaking4.setNotable(true); - breaking4.setTitle("Breaking change number 4"); - breaking4.setArea("Cluster and node setting"); - breaking4.setDetails("Breaking change details 4"); - breaking4.setImpact("Breaking change impact description 4"); - breaking4.setEssSettingChange(true); - - ChangelogEntry entry5 = new ChangelogEntry(); - ChangelogEntry.Deprecation deprecation5 = new ChangelogEntry.Deprecation(); - entry5.setDeprecation(deprecation5); - - deprecation5.setNotable(true); - deprecation5.setTitle("Deprecation change number 5"); - deprecation5.setArea("Cluster and node setting"); - deprecation5.setDetails("Deprecation change details 5"); - deprecation5.setImpact("Deprecation change impact description 5"); - deprecation5.setEssSettingChange(false); - - ChangelogEntry entry6 = new ChangelogEntry(); - ChangelogEntry.Deprecation deprecation6 = new ChangelogEntry.Deprecation(); - entry6.setDeprecation(deprecation6); - - deprecation6.setNotable(true); - deprecation6.setTitle("Deprecation change number 6"); - deprecation6.setArea("Cluster and node setting"); - deprecation6.setDetails("Deprecation change details 6"); - deprecation6.setImpact("Deprecation change impact description 6"); - deprecation6.setEssSettingChange(false); - - ChangelogEntry entry7 = new ChangelogEntry(); - ChangelogEntry.Deprecation deprecation7 = new ChangelogEntry.Deprecation(); - entry7.setDeprecation(deprecation7); - - deprecation7.setNotable(false); - deprecation7.setTitle("Deprecation change number 7"); - deprecation7.setArea("Cluster and node setting"); - deprecation7.setDetails("Deprecation change details 7"); - deprecation7.setImpact("Deprecation change impact description 7"); - deprecation7.setEssSettingChange(false); - - return List.of(entry1, entry2, entry3, entry4, entry5, entry6, entry7); - } - - private String getResource(String name) throws Exception { - return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8); - } -} diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java deleted file mode 100644 index 2a32040035ec7..0000000000000 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; - -public class ReleaseNotesIndexGeneratorTest { - - /** - * Check that a release notes index can be generated. - */ - @Test - public void generateFile_rendersCorrectMarkup() throws Exception { - // given: - final Set versions = Stream.of( - "8.0.0-alpha1", - "8.0.0-beta2", - "8.0.0-rc3", - "8.0.0", - "8.0.1", - "8.0.2", - "8.1.0", - "8.1.1", - "8.2.0-SNAPSHOT" - ).map(QualifiedVersion::of).collect(Collectors.toSet()); - - final String template = getResource("/templates/release-notes-index.asciidoc"); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.generateFile.asciidoc" - ); - - // when: - final String actualOutput = ReleaseNotesIndexGenerator.generateFile(versions, template); - - // then: - assertThat(actualOutput, equalTo(expectedOutput)); - } - - private String getResource(String name) throws Exception { - return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8); - } -} diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc deleted file mode 100644 index ee78d10fd4b07..0000000000000 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc +++ /dev/null @@ -1,134 +0,0 @@ -[[migrating-8.4]] -== Migrating to 8.4 -++++ -8.4 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.4. - -See also <> and <>. - -coming::[8.4.0] - - -[discrete] -[[breaking-changes-8.4]] -=== Breaking changes - -The following changes in {es} 8.4 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.4, review these changes and take the described steps -to mitigate the impact. - -[discrete] -[[breaking_84_api_changes]] -==== API changes - -[[breaking_change_number_1]] -.Breaking change number 1 -[%collapsible] -==== -*Details* + -Breaking change details 1 - -*Impact* + -Breaking change impact description 1 -==== - -[discrete] -[[breaking_84_cluster_and_node_setting_changes]] -==== Cluster and node setting changes - -[[breaking_change_number_2]] -.Breaking change number 2 -[%collapsible] -==== -*Details* + -Breaking change details 2 - -*Impact* + -Breaking change impact description 2 -==== - -[[breaking_change_number_4]] -.Breaking change number 4 -[%collapsible] -==== -*Details* + -Breaking change details 4 - -*Impact* + -Breaking change impact description 4 -==== - -[discrete] -[[breaking_84_transform_changes]] -==== Transform changes - -[[breaking_change_number_3]] -.Breaking change number 3 -[%collapsible] -==== -*Details* + -Breaking change details 3 - -*Impact* + -Breaking change impact description 3 -==== - - -[discrete] -[[deprecated-8.4]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.4 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.4. - -To find out if you are using any deprecated functionality, -enable <>. - -[discrete] -[[deprecations_84_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[deprecation_change_number_5]] -.Deprecation change number 5 -[%collapsible] -==== -*Details* + -Deprecation change details 5 - -*Impact* + -Deprecation change impact description 5 -==== - -[[deprecation_change_number_6]] -.Deprecation change number 6 -[%collapsible] -==== -*Details* + -Deprecation change details 6 - -*Impact* + -Deprecation change impact description 6 -==== - -[discrete] -[[deprecations_84_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[deprecation_change_number_7]] -.Deprecation change number 7 -[%collapsible] -==== -*Details* + -Deprecation change details 7 - -*Impact* + -Deprecation change impact description 7 -==== - diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md new file mode 100644 index 0000000000000..038521f311242 --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md @@ -0,0 +1,22 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes.html +--- + +# Elasticsearch breaking changes [elasticsearch-breaking-changes] +Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. + +To learn how to upgrade, check out . + +% ## Next version [elasticsearch-nextversion-breaking-changes] +% **Release date:** Month day, year + +## 8.2.0 [elasticsearch-820-breaking-changes] +**Release date:** April 01, 2025 + +Aggregation: +* Test changelog entry 0_0 [#0](https://github.com/elastic/elasticsearch/pull/0) (issue: {es-issue}1[#1]) +* Test changelog entry 0_1 [#2](https://github.com/elastic/elasticsearch/pull/2) (issues: {es-issue}3[#3], {es-issue}4[#4]) + + diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md new file mode 100644 index 0000000000000..6d36fc53181e7 --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md @@ -0,0 +1,26 @@ +--- +navigation_title: "Elasticsearch" +--- + +# {{es}} deprecations [elasticsearch-deprecations] +Review the deprecated functionality for your {{es}} version. While deprecations have no immediate impact, we strongly encourage you update your implementation after you upgrade. + +To learn how to upgrade, check out . + +To give you insight into what deprecated features you’re using, {{es}}: + +* Returns a `Warn` HTTP header whenever you submit a request that uses deprecated functionality. +* [Logs deprecation warnings](docs-content://deploy-manage/monitor/logging-configuration/update-elasticsearch-logging-levels.md#deprecation-logging) when deprecated functionality is used. +* [Provides a deprecation info API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-migration-deprecations) that scans a cluster’s configuration and mappings for deprecated functionality. + +% ## Next version [elasticsearch-nextversion-deprecations] +% **Release date:** Month day, year + +## 8.2.0 [elasticsearch-820-deprecations] +**Release date:** April 01, 2025 + +Search: +* Test changelog entry 4_0 [#4000](https://github.com/elastic/elasticsearch/pull/4000) (issue: {es-issue}4001[#4001]) +* Test changelog entry 4_1 [#4002](https://github.com/elastic/elasticsearch/pull/4002) (issues: {es-issue}4003[#4003], {es-issue}4004[#4004]) + + diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.generateFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.generateFile.asciidoc deleted file mode 100644 index 77b00f9ba9852..0000000000000 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.generateFile.asciidoc +++ /dev/null @@ -1,44 +0,0 @@ -[[release-notes-8.2.0]] -== {es} version 8.2.0 - -coming[8.2.0] - -Also see <>. - -[discrete] -[[security-updates-8.2.0]] -=== Security updates - -* Test security issue - -[discrete] -[[known-issues-8.2.0]] -=== Known issues - -* Test known issue - -[[deprecation-8.2.0]] -[float] -=== Deprecations - -Cluster:: -* Test changelog entry 1_0 {es-pull}1000[#1000] (issue: {es-issue}1001[#1001]) -* Test changelog entry 1_1 {es-pull}1002[#1002] (issues: {es-issue}1003[#1003], {es-issue}1004[#1004]) - -[[enhancement-8.2.0]] -[float] -=== Enhancements - -Indices:: -* Test changelog entry 2_0 {es-pull}2000[#2000] (issue: {es-issue}2001[#2001]) -* Test changelog entry 2_1 {es-pull}2002[#2002] (issues: {es-issue}2003[#2003], {es-issue}2004[#2004]) - -[[feature-8.2.0]] -[float] -=== New features - -Mappings:: -* Test changelog entry 3_0 {es-pull}3000[#3000] (issue: {es-issue}3001[#3001]) -* Test changelog entry 3_1 {es-pull}3002[#3002] (issues: {es-issue}3003[#3003], {es-issue}3004[#3004]) - - diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md new file mode 100644 index 0000000000000..803add089461f --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md @@ -0,0 +1,68 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-8.2.0.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.2.html +--- + +# Elasticsearch release notes [elasticsearch-release-notes] + +Review the changes, fixes, and more in each version of Elasticsearch. + +To check for security updates, go to [Security announcements for the Elastic stack](https://discuss.elastic.co/c/announcements/security-announcements/31). + +% Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. + +% ## version.next [elasticsearch-next-release-notes] +% **Release date:** Month day, year + +% ### Features and enhancements [elasticsearch-next-features-enhancements] +% * + +% ### Fixes [elasticsearch-next-fixes] +% * + +## 8.2.0 [elasticsearch-820-release-notes] +**Release date:** April 01, 2025 + +### Features and enhancements [elasticsearch-820-features-enhancements] + +Aggregation: +* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: {es-issue}6001[#6001]) +* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: {es-issue}6003[#6003], {es-issue}6004[#6004]) + +Cluster: +* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: {es-issue}7001[#7001]) +* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: {es-issue}7003[#7003], {es-issue}7004[#7004]) + +Indices: +* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: {es-issue}8001[#8001]) +* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: {es-issue}8003[#8003], {es-issue}8004[#8004]) + +Search: +* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: {es-issue}10001[#10001]) +* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: {es-issue}10003[#10003], {es-issue}10004[#10004]) + +Security: +* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: {es-issue}5001[#5001]) +* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: {es-issue}5003[#5003], {es-issue}5004[#5004]) + +### Fixes [elasticsearch-820-fixes] + +Indices: +* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: {es-issue}2001[#2001]) +* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: {es-issue}2003[#2003], {es-issue}2004[#2004]) + +Mappings: +* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: {es-issue}3001[#3001]) +* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: {es-issue}3003[#3003], {es-issue}3004[#3004]) + +### Regressions [elasticsearch-820-regression] + +Mappings: +* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: {es-issue}9001[#9001]) +* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: {es-issue}9003[#9003], {es-issue}9004[#9004]) + + diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.generateFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.generateFile.asciidoc deleted file mode 100644 index f07a5f21db195..0000000000000 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.generateFile.asciidoc +++ /dev/null @@ -1,30 +0,0 @@ -[[es-release-notes]] -= Release notes - -[partintro] --- - -This section summarizes the changes in each release. - -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> - --- - -include::release-notes/8.2.0.asciidoc[] -include::release-notes/8.1.1.asciidoc[] -include::release-notes/8.1.0.asciidoc[] -include::release-notes/8.0.2.asciidoc[] -include::release-notes/8.0.1.asciidoc[] -include::release-notes/8.0.0.asciidoc[] -include::release-notes/8.0.0-rc3.asciidoc[] -include::release-notes/8.0.0-beta2.asciidoc[] -include::release-notes/8.0.0-alpha1.asciidoc[] - diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index c8456bfca81e6..34b6000ccee32 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -7,7 +7,7 @@ mapped_pages: # Elasticsearch breaking changes [elasticsearch-breaking-changes] Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. -To learn how to upgrade, check out . +To learn how to upgrade, check out . % ## Next version [elasticsearch-nextversion-breaking-changes] % **Release date:** Month day, year diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index 390d605135b9d..009c08c1291a4 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -19,4 +19,10 @@ To give you insight into what deprecated features you’re using, {{es}}: ## 9.1.0 [elasticsearch-910-deprecations] **Release date:** April 01, 2025 -No deprecations in this version. +Machine Learning: +* Add deprecation warning for flush API [#121667](https://github.com/elastic/elasticsearch/pull/121667) (issue: {es-issue}121506[#121506]) + +Search: +* Deprecate Behavioral Analytics CRUD apis [#122960](https://github.com/elastic/elasticsearch/pull/122960) + + diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index c53b866871f5e..b9f43a8d8d8a8 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -15,7 +15,7 @@ To check for security updates, go to [Security announcements for the Elastic sta % Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. -% ## version.next [felasticsearch-next-release-notes] +% ## version.next [elasticsearch-next-release-notes] % **Release date:** Month day, year % ### Features and enhancements [elasticsearch-next-features-enhancements] From 8b1c3b86be1bf719807ac103d50ae2550f627b28 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 12 Mar 2025 16:52:24 -0400 Subject: [PATCH 07/34] Updated release notes and missed file --- .../release/ReleaseNotesGeneratorTest.java | 61 +++++++++++-------- docs/release-notes/index.md | 11 ++++ 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java index cbb53698c586b..1afd2a0204eae 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java @@ -25,15 +25,40 @@ public class ReleaseNotesGeneratorTest { - /** - * Check that the release notes can be correctly generated. - */ + private static final List CHANGE_TYPES = List.of( + "breaking", + "breaking-java", + "bug", + "fixes", + "deprecation", + "enhancement", + "feature", + "features-enhancements", + "new-aggregation", + "regression", + "upgrade" + ); + + @Test + public void generateFile_index_rendersCorrectMarkup() throws Exception { + testTemplate("index.md"); + } + + @Test + public void generateFile_breakingChanges_rendersCorrectMarkup() throws Exception { + testTemplate("breaking-changes.md"); + } + @Test - public void generateFile_rendersCorrectMarkup() throws Exception { + public void generateFile_deprecations_rendersCorrectMarkup() throws Exception { + testTemplate("deprecations.md"); + } + + public void testTemplate(String templateFilename) throws Exception { // given: - final String template = getResource("/templates/release-notes.asciidoc"); + final String template = getResource("/templates/" + templateFilename); final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.generateFile.asciidoc" + "/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest." + templateFilename ); final Set entries = getEntries(); @@ -47,23 +72,9 @@ public void generateFile_rendersCorrectMarkup() throws Exception { private Set getEntries() { final Set entries = new HashSet<>(); - entries.addAll(buildEntries(1, 2)); - entries.addAll(buildEntries(2, 2)); - entries.addAll(buildEntries(3, 2)); - - // Security issues are presented first in the notes - final ChangelogEntry securityEntry = new ChangelogEntry(); - securityEntry.setArea("Security"); - securityEntry.setType("security"); - securityEntry.setSummary("Test security issue"); - entries.add(securityEntry); - - // known issues are presented after security issues - final ChangelogEntry knownIssue = new ChangelogEntry(); - knownIssue.setArea("Search"); - knownIssue.setType("known-issue"); - knownIssue.setSummary("Test known issue"); - entries.add(knownIssue); + for (int i = 0; i < CHANGE_TYPES.size(); i++) { + entries.addAll(buildEntries(i, 2)); + } return entries; } @@ -71,11 +82,9 @@ private Set getEntries() { private List buildEntries(int seed, int count) { // Sample of possible areas from `changelog-schema.json` final List areas = List.of("Aggregation", "Cluster", "Indices", "Mappings", "Search", "Security"); - // Possible change types, with `breaking`, `breaking-java`, `known-issue` and `security` removed. - final List types = List.of("bug", "deprecation", "enhancement", "feature", "new-aggregation", "regression", "upgrade"); final String area = areas.get(seed % areas.size()); - final String type = types.get(seed % types.size()); + final String type = CHANGE_TYPES.get(seed % CHANGE_TYPES.size()); final List entries = new ArrayList<>(count); diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index b9f43a8d8d8a8..6474f6ba56e4f 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -50,11 +50,13 @@ ES|QL: * Allow skip shards with `_tier` and `_index` in ES|QL [#123728](https://github.com/elastic/elasticsearch/pull/123728) * Avoid `NamedWritable` in block serialization [#124394](https://github.com/elastic/elasticsearch/pull/124394) * ES|QL - Add scoring for full text functions disjunctions [#121793](https://github.com/elastic/elasticsearch/pull/121793) +* ES|QL: Support `::date` in inline cast [#123460](https://github.com/elastic/elasticsearch/pull/123460) (issue: {es-issue}116746[#116746]) * Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: {es-issue}122967[#122967]) * Implement runtime skip_unavailable=true [#121240](https://github.com/elastic/elasticsearch/pull/121240) * Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) * Introduce `allow_partial_results` setting in ES|QL [#122890](https://github.com/elastic/elasticsearch/pull/122890) * Introduce a pre-mapping logical plan processing step [#121260](https://github.com/elastic/elasticsearch/pull/121260) +* Pragma to load from stored fields [#122891](https://github.com/elastic/elasticsearch/pull/122891) * Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: {es-issue}123067[#123067]) * Render `aggregate_metric_double` [#122660](https://github.com/elastic/elasticsearch/pull/122660) * Retry ES|QL node requests on shard level failures [#120774](https://github.com/elastic/elasticsearch/pull/120774) @@ -86,6 +88,8 @@ Machine Learning: * Adding support for binary embedding type to Cohere service embedding type [#120751](https://github.com/elastic/elasticsearch/pull/120751) * Adding support for specifying embedding type to Jina AI service settings [#121548](https://github.com/elastic/elasticsearch/pull/121548) * ES|QL `change_point` processing command [#120998](https://github.com/elastic/elasticsearch/pull/120998) +* Integrate with `DeepSeek` API [#122218](https://github.com/elastic/elasticsearch/pull/122218) +* [Inference API] Propagate product use case http header to EIS [#124025](https://github.com/elastic/elasticsearch/pull/124025) Mapping: * Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. [#122615](https://github.com/elastic/elasticsearch/pull/122615) (issue: {es-issue}116726[#116726]) @@ -110,6 +114,7 @@ Security: * Bump nimbus-jose-jwt to 10.0.2 [#124544](https://github.com/elastic/elasticsearch/pull/124544) Snapshot/Restore: +* GCS blob store: add `OperationPurpose/Operation` stats counters [#122991](https://github.com/elastic/elasticsearch/pull/122991) * Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) Stats: @@ -170,6 +175,7 @@ Infra/Core: Ingest Node: * Fix geoip databases index access after system feature migration (again) [#122938](https://github.com/elastic/elasticsearch/pull/122938) +* Fix geoip databases index access after system feature migration (take 3) [#124604](https://github.com/elastic/elasticsearch/pull/124604) * apm-data: Use representative count as event.success_count if available [#119995](https://github.com/elastic/elasticsearch/pull/119995) Machine Learning: @@ -177,6 +183,7 @@ Machine Learning: * Add enterprise license check to inference action for semantic text fields [#122293](https://github.com/elastic/elasticsearch/pull/122293) * Avoid potentially throwing calls to Task#getDescription in model download [#124527](https://github.com/elastic/elasticsearch/pull/124527) * Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) +* Provide model size statistics as soon as an anomaly detection job is opened [#124638](https://github.com/elastic/elasticsearch/pull/124638) (issue: {es-issue}121168[#121168]) * Retry on streaming errors [#123076](https://github.com/elastic/elasticsearch/pull/123076) * Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) * Updates to allow using Cohere binary embedding response in semantic search queries [#121827](https://github.com/elastic/elasticsearch/pull/121827) @@ -188,11 +195,15 @@ Mapping: Ranking: * Restore `TextSimilarityRankBuilder` XContent output [#124564](https://github.com/elastic/elasticsearch/pull/124564) +Relevance: +* Prevent Query Rule Creation with Invalid Numeric Match Criteria [#122823](https://github.com/elastic/elasticsearch/pull/122823) + Search: * Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions [#123890](https://github.com/elastic/elasticsearch/pull/123890) * Fix concurrency issue in `ScriptSortBuilder` [#123757](https://github.com/elastic/elasticsearch/pull/123757) * Fix handling of auto expand replicas for stateless indices [#122365](https://github.com/elastic/elasticsearch/pull/122365) * Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: {es-issue}122186[#122186]) +* Let MLTQuery throw IAE when no analyzer is set [#124662](https://github.com/elastic/elasticsearch/pull/124662) (issue: {es-issue}124562[#124562]) Snapshot/Restore: * Fork post-snapshot-delete cleanup off master thread [#122731](https://github.com/elastic/elasticsearch/pull/122731) From a53b29a3cd582937dd015f81972983ad118af610 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 13 Mar 2025 15:11:38 -0400 Subject: [PATCH 08/34] wip adding release highlights --- .../release/ReleaseNotesGenerator.java | 13 +++++++++++++ .../src/main/resources/templates/index.md | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java index 3a14bdb74e04c..c9fc64171d09e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java @@ -18,10 +18,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import static java.util.Comparator.comparing; +import static java.util.Comparator.comparingInt; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; @@ -66,12 +68,23 @@ static void update(File templateFile, File outputFile, QualifiedVersion version, static String generateFile(String template, QualifiedVersion version, Set changelogs) throws IOException { final var changelogsByTypeByArea = buildChangelogBreakdown(changelogs); + final Map> groupedHighlights = changelogs.stream() + .map(ChangelogEntry::getHighlight) + .filter(Objects::nonNull) + .sorted(comparingInt(ChangelogEntry.Highlight::getPr)) + .collect(groupingBy(ChangelogEntry.Highlight::isNotable, toList())); + + final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); + final List nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); + final Map bindings = new HashMap<>(); bindings.put("version", version); bindings.put("changelogsByTypeByArea", changelogsByTypeByArea); bindings.put("TYPE_LABELS", TYPE_LABELS); bindings.put("unqualifiedVersion", version.withoutQualifier()); bindings.put("versionWithoutSeparator", version.withoutQualifier().toString().replaceAll("\\.", "")); + bindings.put("notableHighlights", notableHighlights); + bindings.put("nonNotableHighlights", nonNotableHighlights); return TemplateUtils.render(template, bindings); } diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md index bb6260c3b285b..c02e51263b10e 100644 --- a/build-tools-internal/src/main/resources/templates/index.md +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -26,7 +26,24 @@ To check for security updates, go to [Security announcements for the Elastic sta ## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-release-notes] **Release date:** April 01, 2025 + <% +if (!notableHighlights.isEmpty() || !nonNotableHighlights.isEmpty()) { + print "### Highlights\n" +} + +for (highlights in [notableHighlights, nonNotableHighlights]) { + if (!highlights.isEmpty()) { + for (highlight in highlights) { %> +::::{dropdown} ${highlight.title} +${highlight.body.trim()} + +For more information, check [PR #${highlight.pr}](https://github.com/elastic/elasticsearch/pull/${highlight.pr}). +:::: +<% } + } +} + for (changeType in ['features-enhancements', 'fixes', 'regression']) { if (changelogsByTypeByArea[changeType] == null || changelogsByTypeByArea[changeType].empty) { continue; @@ -40,7 +57,7 @@ for (changeType in ['features-enhancements', 'fixes', 'regression']) { print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" if (change.issues != null && change.issues.empty == false) { print change.issues.size() == 1 ? " (issue: " : " (issues: " - print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") + print change.issues.collect { "[#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" }.join(", ") print ")" } print "\n" From 54873b4fe94a0e9ae140431377594d6309162b4b Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 20 Mar 2025 17:55:35 -0400 Subject: [PATCH 09/34] Fix up release highlights, add tests --- .../src/main/resources/templates/index.md | 3 +- .../release/ReleaseNotesGeneratorTest.java | 60 +++++++- .../ReleaseNotesGeneratorTest.index.md | 55 +++++--- ...easeNotesGeneratorTest.index.no-changes.md | 30 ++++ ...eNotesGeneratorTest.index.no-highlights.md | 68 +++++++++ docs/release-notes/breaking-changes.md | 3 + docs/release-notes/index.md | 132 ++++++++++++++---- 7 files changed, 303 insertions(+), 48 deletions(-) create mode 100644 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md create mode 100644 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md index c02e51263b10e..a17b86bd3cd06 100644 --- a/build-tools-internal/src/main/resources/templates/index.md +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -26,10 +26,9 @@ To check for security updates, go to [Security announcements for the Elastic sta ## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-release-notes] **Release date:** April 01, 2025 - <% if (!notableHighlights.isEmpty() || !nonNotableHighlights.isEmpty()) { - print "### Highlights\n" + print "\n### Highlights [elasticsearch-${versionWithoutSeparator}-highlights]\n" } for (highlights in [notableHighlights, nonNotableHighlights]) { diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java index 1afd2a0204eae..96b8a32a0745f 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThat; @@ -44,6 +45,21 @@ public void generateFile_index_rendersCorrectMarkup() throws Exception { testTemplate("index.md"); } + @Test + public void generateFile_index_noHighlights_rendersCorrectMarkup() throws Exception { + Set entries = getEntries(); + entries = entries.stream().filter(e -> e.getHighlight() == null).collect(Collectors.toSet()); + + testTemplate("index.md", "index.no-highlights.md", entries); + } + + @Test + public void generateFile_index_noChanges_rendersCorrectMarkup() throws Exception { + Set entries = new HashSet<>(); + + testTemplate("index.md", "index.no-changes.md", entries); + } + @Test public void generateFile_breakingChanges_rendersCorrectMarkup() throws Exception { testTemplate("breaking-changes.md"); @@ -55,13 +71,21 @@ public void generateFile_deprecations_rendersCorrectMarkup() throws Exception { } public void testTemplate(String templateFilename) throws Exception { + testTemplate(templateFilename, templateFilename, null); + } + + public void testTemplate(String templateFilename, String outputFilename) throws Exception { + testTemplate(templateFilename, outputFilename, null); + } + + public void testTemplate(String templateFilename, String outputFilename, Set entries) throws Exception { // given: final String template = getResource("/templates/" + templateFilename); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest." + templateFilename - ); + final String expectedOutput = getResource("/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest." + outputFilename); - final Set entries = getEntries(); + if (entries == null) { + entries = getEntries(); + } // when: final String actualOutput = ReleaseNotesGenerator.generateFile(template, QualifiedVersion.of("8.2.0-SNAPSHOT"), entries); @@ -76,6 +100,10 @@ private Set getEntries() { entries.addAll(buildEntries(i, 2)); } + entries.add(makeHighlightsEntry(5001, false)); + entries.add(makeHighlightsEntry(5000, true)); + entries.add(makeHighlightsEntry(5002, true)); + return entries; } @@ -110,6 +138,30 @@ private List buildEntries(int seed, int count) { return entries; } + private List getHighlightsEntries() { + ChangelogEntry entry123 = makeHighlightsEntry(123, true); + ChangelogEntry entry456 = makeHighlightsEntry(456, true); + ChangelogEntry entry789 = makeHighlightsEntry(789, false); + // Return unordered list, to test correct re-ordering + return List.of(entry456, entry123, entry789); + } + + private ChangelogEntry makeHighlightsEntry(int pr, boolean notable) { + ChangelogEntry entry = new ChangelogEntry(); + entry.setPr(pr); + ChangelogEntry.Highlight highlight = new ChangelogEntry.Highlight(); + entry.setHighlight(highlight); + + highlight.setNotable(notable); + highlight.setTitle((notable ? "[Notable] " : "") + "Release highlight number " + pr); + highlight.setBody("Release highlight body number " + pr); + entry.setType("feature"); + entry.setArea("Search"); + entry.setSummary(""); + + return entry; + } + private String getResource(String name) throws Exception { return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8); } diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md index 803add089461f..dc4320f684535 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md @@ -27,42 +27,65 @@ To check for security updates, go to [Security announcements for the Elastic sta ## 8.2.0 [elasticsearch-820-release-notes] **Release date:** April 01, 2025 +### Highlights [elasticsearch-820-highlights] + +::::{dropdown} [Notable] Release highlight number 5000 +Release highlight body number 5000 + +For more information, check [PR #5000](https://github.com/elastic/elasticsearch/pull/5000). +:::: + +::::{dropdown} [Notable] Release highlight number 5002 +Release highlight body number 5002 + +For more information, check [PR #5002](https://github.com/elastic/elasticsearch/pull/5002). +:::: + +::::{dropdown} Release highlight number 5001 +Release highlight body number 5001 + +For more information, check [PR #5001](https://github.com/elastic/elasticsearch/pull/5001). +:::: + ### Features and enhancements [elasticsearch-820-features-enhancements] Aggregation: -* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: {es-issue}6001[#6001]) -* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: {es-issue}6003[#6003], {es-issue}6004[#6004]) +* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: [#6000](https://github.com/elastic/elasticsearch/pull/6000)) +* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: [#6002](https://github.com/elastic/elasticsearch/pull/6002), [#6002](https://github.com/elastic/elasticsearch/pull/6002)) Cluster: -* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: {es-issue}7001[#7001]) -* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: {es-issue}7003[#7003], {es-issue}7004[#7004]) +* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: [#7000](https://github.com/elastic/elasticsearch/pull/7000)) +* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: [#7002](https://github.com/elastic/elasticsearch/pull/7002), [#7002](https://github.com/elastic/elasticsearch/pull/7002)) Indices: -* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: {es-issue}8001[#8001]) -* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: {es-issue}8003[#8003], {es-issue}8004[#8004]) +* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: [#8000](https://github.com/elastic/elasticsearch/pull/8000)) +* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: [#8002](https://github.com/elastic/elasticsearch/pull/8002), [#8002](https://github.com/elastic/elasticsearch/pull/8002)) Search: -* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: {es-issue}10001[#10001]) -* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: {es-issue}10003[#10003], {es-issue}10004[#10004]) +* [#5002](https://github.com/elastic/elasticsearch/pull/5002) +* [#5000](https://github.com/elastic/elasticsearch/pull/5000) +* [#5001](https://github.com/elastic/elasticsearch/pull/5001) +* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: [#10000](https://github.com/elastic/elasticsearch/pull/10000)) +* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: [#10002](https://github.com/elastic/elasticsearch/pull/10002), [#10002](https://github.com/elastic/elasticsearch/pull/10002)) Security: -* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: {es-issue}5001[#5001]) -* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: {es-issue}5003[#5003], {es-issue}5004[#5004]) +* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: [#5000](https://github.com/elastic/elasticsearch/pull/5000)) +* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: [#5002](https://github.com/elastic/elasticsearch/pull/5002), [#5002](https://github.com/elastic/elasticsearch/pull/5002)) ### Fixes [elasticsearch-820-fixes] Indices: -* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: {es-issue}2001[#2001]) -* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: {es-issue}2003[#2003], {es-issue}2004[#2004]) +* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: [#2000](https://github.com/elastic/elasticsearch/pull/2000)) +* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: [#2002](https://github.com/elastic/elasticsearch/pull/2002), [#2002](https://github.com/elastic/elasticsearch/pull/2002)) Mappings: -* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: {es-issue}3001[#3001]) -* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: {es-issue}3003[#3003], {es-issue}3004[#3004]) +* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: [#3000](https://github.com/elastic/elasticsearch/pull/3000)) +* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: [#3002](https://github.com/elastic/elasticsearch/pull/3002), [#3002](https://github.com/elastic/elasticsearch/pull/3002)) ### Regressions [elasticsearch-820-regression] Mappings: -* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: {es-issue}9001[#9001]) -* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: {es-issue}9003[#9003], {es-issue}9004[#9004]) +* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: [#9000](https://github.com/elastic/elasticsearch/pull/9000)) +* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: [#9002](https://github.com/elastic/elasticsearch/pull/9002), [#9002](https://github.com/elastic/elasticsearch/pull/9002)) diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md new file mode 100644 index 0000000000000..ea2b62653ae8b --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md @@ -0,0 +1,30 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-8.2.0.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.2.html +--- + +# Elasticsearch release notes [elasticsearch-release-notes] + +Review the changes, fixes, and more in each version of Elasticsearch. + +To check for security updates, go to [Security announcements for the Elastic stack](https://discuss.elastic.co/c/announcements/security-announcements/31). + +% Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. + +% ## version.next [elasticsearch-next-release-notes] +% **Release date:** Month day, year + +% ### Features and enhancements [elasticsearch-next-features-enhancements] +% * + +% ### Fixes [elasticsearch-next-fixes] +% * + +## 8.2.0 [elasticsearch-820-release-notes] +**Release date:** April 01, 2025 + + diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md new file mode 100644 index 0000000000000..831c21251bbd1 --- /dev/null +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md @@ -0,0 +1,68 @@ +--- +navigation_title: "Elasticsearch" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-8.2.0.html + - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.2.html +--- + +# Elasticsearch release notes [elasticsearch-release-notes] + +Review the changes, fixes, and more in each version of Elasticsearch. + +To check for security updates, go to [Security announcements for the Elastic stack](https://discuss.elastic.co/c/announcements/security-announcements/31). + +% Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. + +% ## version.next [elasticsearch-next-release-notes] +% **Release date:** Month day, year + +% ### Features and enhancements [elasticsearch-next-features-enhancements] +% * + +% ### Fixes [elasticsearch-next-fixes] +% * + +## 8.2.0 [elasticsearch-820-release-notes] +**Release date:** April 01, 2025 + +### Features and enhancements [elasticsearch-820-features-enhancements] + +Aggregation: +* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: [#6000](https://github.com/elastic/elasticsearch/pull/6000)) +* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: [#6002](https://github.com/elastic/elasticsearch/pull/6002), [#6002](https://github.com/elastic/elasticsearch/pull/6002)) + +Cluster: +* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: [#7000](https://github.com/elastic/elasticsearch/pull/7000)) +* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: [#7002](https://github.com/elastic/elasticsearch/pull/7002), [#7002](https://github.com/elastic/elasticsearch/pull/7002)) + +Indices: +* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: [#8000](https://github.com/elastic/elasticsearch/pull/8000)) +* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: [#8002](https://github.com/elastic/elasticsearch/pull/8002), [#8002](https://github.com/elastic/elasticsearch/pull/8002)) + +Search: +* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: [#10000](https://github.com/elastic/elasticsearch/pull/10000)) +* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: [#10002](https://github.com/elastic/elasticsearch/pull/10002), [#10002](https://github.com/elastic/elasticsearch/pull/10002)) + +Security: +* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: [#5000](https://github.com/elastic/elasticsearch/pull/5000)) +* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: [#5002](https://github.com/elastic/elasticsearch/pull/5002), [#5002](https://github.com/elastic/elasticsearch/pull/5002)) + +### Fixes [elasticsearch-820-fixes] + +Indices: +* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: [#2000](https://github.com/elastic/elasticsearch/pull/2000)) +* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: [#2002](https://github.com/elastic/elasticsearch/pull/2002), [#2002](https://github.com/elastic/elasticsearch/pull/2002)) + +Mappings: +* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: [#3000](https://github.com/elastic/elasticsearch/pull/3000)) +* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: [#3002](https://github.com/elastic/elasticsearch/pull/3002), [#3002](https://github.com/elastic/elasticsearch/pull/3002)) + +### Regressions [elasticsearch-820-regression] + +Mappings: +* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: [#9000](https://github.com/elastic/elasticsearch/pull/9000)) +* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: [#9002](https://github.com/elastic/elasticsearch/pull/9002), [#9002](https://github.com/elastic/elasticsearch/pull/9002)) + + diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index 34b6000ccee32..822f01b979b32 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -15,6 +15,9 @@ To learn how to upgrade, check out . ## 9.1.0 [elasticsearch-910-breaking-changes] **Release date:** April 01, 2025 +Discovery-Plugins: +* Upgrade `discovery-ec2` to AWS SDK v2 [#122062](https://github.com/elastic/elasticsearch/pull/122062) + TLS: * Drop `TLS_RSA` cipher support for JDK 24 [#123600](https://github.com/elastic/elasticsearch/pull/123600) diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 6474f6ba56e4f..c10b6623e6228 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -27,6 +27,14 @@ To check for security updates, go to [Security announcements for the Elastic sta ## 9.1.0 [elasticsearch-910-release-notes] **Release date:** April 01, 2025 +### Highlights + +::::{dropdown} Release semantic_text as a GA feature +semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. + +For more information, check [PR #124669](https://github.com/elastic/elasticsearch/pull/124669). +:::: + ### Features and enhancements [elasticsearch-910-features-enhancements] Allocation: @@ -40,28 +48,45 @@ CRUD: Data streams: * Add index mode to get data stream API [#122486](https://github.com/elastic/elasticsearch/pull/122486) +* Reindex data stream indices on different nodes [#125171](https://github.com/elastic/elasticsearch/pull/125171) * Retry ILM async action after reindexing data stream [#124149](https://github.com/elastic/elasticsearch/pull/124149) +* Run `TransportGetDataStreamsAction` on local node [#122852](https://github.com/elastic/elasticsearch/pull/122852) * Set cause on create index request in create from action [#124363](https://github.com/elastic/elasticsearch/pull/124363) +Downsampling: +* Improve downsample performance by buffering docids and do bulk processing [#124477](https://github.com/elastic/elasticsearch/pull/124477) +* Improve rolling up metrics [#124739](https://github.com/elastic/elasticsearch/pull/124739) + ES|QL: * Add initial grammar and changes for FORK [#121948](https://github.com/elastic/elasticsearch/pull/121948) * Add initial grammar and planning for RRF (snapshot) [#123396](https://github.com/elastic/elasticsearch/pull/123396) * Allow partial results in ES|QL [#121942](https://github.com/elastic/elasticsearch/pull/121942) * Allow skip shards with `_tier` and `_index` in ES|QL [#123728](https://github.com/elastic/elasticsearch/pull/123728) * Avoid `NamedWritable` in block serialization [#124394](https://github.com/elastic/elasticsearch/pull/124394) +* Double parameter markers for identifiers [#122459](https://github.com/elastic/elasticsearch/pull/122459) +* ESQL: Align `RENAME` behavior with `EVAL` for sequential processing [#122250](https://github.com/elastic/elasticsearch/pull/122250) (issue: [#122250](https://github.com/elastic/elasticsearch/pull/122250)) * ES|QL - Add scoring for full text functions disjunctions [#121793](https://github.com/elastic/elasticsearch/pull/121793) -* ES|QL: Support `::date` in inline cast [#123460](https://github.com/elastic/elasticsearch/pull/123460) (issue: {es-issue}116746[#116746]) -* Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: {es-issue}122967[#122967]) +* ES|QL slow log [#124094](https://github.com/elastic/elasticsearch/pull/124094) +* ES|QL: Support `::date` in inline cast [#123460](https://github.com/elastic/elasticsearch/pull/123460) (issue: [#123460](https://github.com/elastic/elasticsearch/pull/123460)) +* Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: [#123290](https://github.com/elastic/elasticsearch/pull/123290)) * Implement runtime skip_unavailable=true [#121240](https://github.com/elastic/elasticsearch/pull/121240) +* Include failures in partial response [#124929](https://github.com/elastic/elasticsearch/pull/124929) * Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) * Introduce `allow_partial_results` setting in ES|QL [#122890](https://github.com/elastic/elasticsearch/pull/122890) * Introduce a pre-mapping logical plan processing step [#121260](https://github.com/elastic/elasticsearch/pull/121260) * Pragma to load from stored fields [#122891](https://github.com/elastic/elasticsearch/pull/122891) -* Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: {es-issue}123067[#123067]) +* Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: [#123381](https://github.com/elastic/elasticsearch/pull/123381)) +* Remove page alignment in exchange sink [#124610](https://github.com/elastic/elasticsearch/pull/124610) * Render `aggregate_metric_double` [#122660](https://github.com/elastic/elasticsearch/pull/122660) +* Report failures on partial results [#124823](https://github.com/elastic/elasticsearch/pull/124823) * Retry ES|QL node requests on shard level failures [#120774](https://github.com/elastic/elasticsearch/pull/120774) +* Reuse child `outputSet` inside the plan where possible [#124611](https://github.com/elastic/elasticsearch/pull/124611) * Support partial results in CCS in ES|QL [#122708](https://github.com/elastic/elasticsearch/pull/122708) * Support subset of metrics in aggregate metric double [#121805](https://github.com/elastic/elasticsearch/pull/121805) +* `ToAggregateMetricDouble` function [#124595](https://github.com/elastic/elasticsearch/pull/124595) + +Engine: +* Threadpool merge scheduler [#120869](https://github.com/elastic/elasticsearch/pull/120869) Health: * Add health indicator impact to `HealthPeriodicLogger` [#122390](https://github.com/elastic/elasticsearch/pull/122390) @@ -69,45 +94,73 @@ Health: ILM+SLM: * Improve SLM Health Indicator to cover missing snapshot [#121370](https://github.com/elastic/elasticsearch/pull/121370) * Run `TransportExplainLifecycleAction` on local node [#122885](https://github.com/elastic/elasticsearch/pull/122885) +* Truncate `step_info` and error reason in ILM execution state and history [#125054](https://github.com/elastic/elasticsearch/pull/125054) (issue: [#125054](https://github.com/elastic/elasticsearch/pull/125054)) + +Indices APIs: +* Run `TransportGetMappingsAction` on local node [#122921](https://github.com/elastic/elasticsearch/pull/122921) Inference: * [Inference API] Rename `model_id` prop to model in EIS sparse inference request body [#122272](https://github.com/elastic/elasticsearch/pull/122272) +Infra/CLI: +* Ignore _JAVA_OPTIONS [#124843](https://github.com/elastic/elasticsearch/pull/124843) + Infra/Core: * Give Kibana user 'all' permissions for .entity_analytics.* indices [#123588](https://github.com/elastic/elasticsearch/pull/123588) * Improve size limiting string message [#122427](https://github.com/elastic/elasticsearch/pull/122427) +* Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#125117](https://github.com/elastic/elasticsearch/pull/125117) + +Infra/REST API: +* Indicate when errors represent timeouts [#124936](https://github.com/elastic/elasticsearch/pull/124936) + +Infra/Settings: +* Allow passing several reserved state chunks in single process call [#124574](https://github.com/elastic/elasticsearch/pull/124574) Ingest Node: -* Allow setting the `type` in the reroute processor [#122409](https://github.com/elastic/elasticsearch/pull/122409) (issue: {es-issue}121553[#121553]) +* Allow setting the `type` in the reroute processor [#122409](https://github.com/elastic/elasticsearch/pull/122409) (issue: [#122409](https://github.com/elastic/elasticsearch/pull/122409)) * Run `TransportEnrichStatsAction` on local node [#121256](https://github.com/elastic/elasticsearch/pull/121256) Machine Learning: +* Add `ModelRegistryMetadata` to Cluster State [#121106](https://github.com/elastic/elasticsearch/pull/121106) * Adding elser default endpoint for EIS [#122066](https://github.com/elastic/elasticsearch/pull/122066) * Adding endpoint creation validation to `ElasticInferenceService` [#117642](https://github.com/elastic/elasticsearch/pull/117642) * Adding integration for VoyageAI embeddings and rerank models [#122134](https://github.com/elastic/elasticsearch/pull/122134) * Adding support for binary embedding type to Cohere service embedding type [#120751](https://github.com/elastic/elasticsearch/pull/120751) * Adding support for specifying embedding type to Jina AI service settings [#121548](https://github.com/elastic/elasticsearch/pull/121548) * ES|QL `change_point` processing command [#120998](https://github.com/elastic/elasticsearch/pull/120998) +* Expose `input_type` option at root level for `text_embedding` task type in Perform Inference API [#122638](https://github.com/elastic/elasticsearch/pull/122638) (issue: [#122638](https://github.com/elastic/elasticsearch/pull/122638)) * Integrate with `DeepSeek` API [#122218](https://github.com/elastic/elasticsearch/pull/122218) +* Limit the number of chunks for semantic text to prevent high memory usage [#123150](https://github.com/elastic/elasticsearch/pull/123150) +* Upgrade AWS v2 SDK to 2.30.38 [#124738](https://github.com/elastic/elasticsearch/pull/124738) * [Inference API] Propagate product use case http header to EIS [#124025](https://github.com/elastic/elasticsearch/pull/124025) Mapping: -* Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. [#122615](https://github.com/elastic/elasticsearch/pull/122615) (issue: {es-issue}116726[#116726]) -* Enable the use of nested field type with index.mode=time_series [#122224](https://github.com/elastic/elasticsearch/pull/122224) (issue: {es-issue}120874[#120874]) +* Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. [#122615](https://github.com/elastic/elasticsearch/pull/122615) (issue: [#122615](https://github.com/elastic/elasticsearch/pull/122615)) +* Enable the use of nested field type with index.mode=time_series [#122224](https://github.com/elastic/elasticsearch/pull/122224) (issue: [#122224](https://github.com/elastic/elasticsearch/pull/122224)) * Improved error message when index field type is invalid [#122860](https://github.com/elastic/elasticsearch/pull/122860) * Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields [#119546](https://github.com/elastic/elasticsearch/pull/119546) +* Release semantic_text as a GA feature [#124669](https://github.com/elastic/elasticsearch/pull/124669) * Store arrays offsets for ip fields natively with synthetic source [#122999](https://github.com/elastic/elasticsearch/pull/122999) * Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. [#113757](https://github.com/elastic/elasticsearch/pull/113757) +* Store arrays offsets for numeric fields natively with synthetic source [#124594](https://github.com/elastic/elasticsearch/pull/124594) +* Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` [#124927](https://github.com/elastic/elasticsearch/pull/124927) * Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields [#122637](https://github.com/elastic/elasticsearch/pull/122637) * Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields [#124050](https://github.com/elastic/elasticsearch/pull/124050) * Use `FallbackSyntheticSourceBlockLoader` for number fields [#122280](https://github.com/elastic/elasticsearch/pull/122280) +Packaging: +* Update bundled JDK to Java 24 [#125159](https://github.com/elastic/elasticsearch/pull/125159) + +Ranking: +* Leverage scorer supplier in `QueryFeatureExtractor` [#125259](https://github.com/elastic/elasticsearch/pull/125259) + Relevance: * Skip semantic_text embedding generation when no content is provided. [#123763](https://github.com/elastic/elasticsearch/pull/123763) Search: -* Account for the `SearchHit` source in circuit breaker [#121920](https://github.com/elastic/elasticsearch/pull/121920) (issue: {es-issue}89656[#89656]) -* Added optional parameters to QSTR ES|QL function [#121787](https://github.com/elastic/elasticsearch/pull/121787) (issue: {es-issue}120933[#120933]) +* Account for the `SearchHit` source in circuit breaker [#121920](https://github.com/elastic/elasticsearch/pull/121920) (issue: [#121920](https://github.com/elastic/elasticsearch/pull/121920)) +* Added optional parameters to QSTR ES|QL function [#121787](https://github.com/elastic/elasticsearch/pull/121787) (issue: [#121787](https://github.com/elastic/elasticsearch/pull/121787)) +* Optimize memory usage in `ShardBulkInferenceActionFilter` [#124313](https://github.com/elastic/elasticsearch/pull/124313) * Optionally allow text similarity reranking to fail [#121784](https://github.com/elastic/elasticsearch/pull/121784) Security: @@ -115,6 +168,7 @@ Security: Snapshot/Restore: * GCS blob store: add `OperationPurpose/Operation` stats counters [#122991](https://github.com/elastic/elasticsearch/pull/122991) +* Retry when the server can't be resolved (Google Cloud Storage) [#123852](https://github.com/elastic/elasticsearch/pull/123852) * Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) Stats: @@ -125,7 +179,8 @@ Store: Vector Search: * Add bit vector support to semantic text [#123187](https://github.com/elastic/elasticsearch/pull/123187) -* Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields [#122381](https://github.com/elastic/elasticsearch/pull/122381) (issue: {es-issue}117274[#117274]) +* Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields [#122381](https://github.com/elastic/elasticsearch/pull/122381) (issue: [#122381](https://github.com/elastic/elasticsearch/pull/122381)) +* New `vector_rescore` parameter as a quantized index type option [#124581](https://github.com/elastic/elasticsearch/pull/124581) Watcher: * Run `TransportGetWatcherSettingsAction` on local node [#122857](https://github.com/elastic/elasticsearch/pull/122857) @@ -138,38 +193,57 @@ Allocation: Analysis: * Adjust exception thrown when unable to load hunspell dict [#123743](https://github.com/elastic/elasticsearch/pull/123743) +Cluster Coordination: +* Disable logging in `ClusterFormationFailureHelper` on shutdown [#125244](https://github.com/elastic/elasticsearch/pull/125244) (issue: [#125244](https://github.com/elastic/elasticsearch/pull/125244)) + Data streams: * Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover [#122905](https://github.com/elastic/elasticsearch/pull/122905) +Distributed: +* Pass `IndexReshardingMetadata` over the wire [#124841](https://github.com/elastic/elasticsearch/pull/124841) + Downsampling: * Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. [#124451](https://github.com/elastic/elasticsearch/pull/124451) +EQL: +* Fix EQL double invoking listener [#124918](https://github.com/elastic/elasticsearch/pull/124918) + ES|QL: -* Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: {es-issue}122413[#122413]) +* Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: [#122886](https://github.com/elastic/elasticsearch/pull/122886)) * Avoid over collecting in Limit or Lucene Operator [#123296](https://github.com/elastic/elasticsearch/pull/123296) -* ESQL: Fix inconsistent results in using scaled_float field [#122586](https://github.com/elastic/elasticsearch/pull/122586) (issue: {es-issue}122547[#122547]) -* ESQL: Remove estimated row size assertion [#122762](https://github.com/elastic/elasticsearch/pull/122762) (issue: {es-issue}121535[#121535]) +* Catch parsing exception [#124958](https://github.com/elastic/elasticsearch/pull/124958) (issue: [#124958](https://github.com/elastic/elasticsearch/pull/124958)) +* Change the order of the optimization rules [#124335](https://github.com/elastic/elasticsearch/pull/124335) +* ESQL: Fix inconsistent results in using scaled_float field [#122586](https://github.com/elastic/elasticsearch/pull/122586) (issue: [#122586](https://github.com/elastic/elasticsearch/pull/122586)) +* ESQL: Remove estimated row size assertion [#122762](https://github.com/elastic/elasticsearch/pull/122762) (issue: [#122762](https://github.com/elastic/elasticsearch/pull/122762)) +* ES|QL: Fix scoring for full text functions [#124540](https://github.com/elastic/elasticsearch/pull/124540) * Fix early termination in `LuceneSourceOperator` [#123197](https://github.com/elastic/elasticsearch/pull/123197) -* Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: {es-issue}123430[#123430]) -* Fix functions emitting warnings with no source [#122821](https://github.com/elastic/elasticsearch/pull/122821) (issue: {es-issue}122588[#122588]) -* Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: {es-issue}121890[#121890]) +* Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: [#123492](https://github.com/elastic/elasticsearch/pull/123492)) +* Fix functions emitting warnings with no source [#122821](https://github.com/elastic/elasticsearch/pull/122821) (issue: [#122821](https://github.com/elastic/elasticsearch/pull/122821)) +* Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: [#122601](https://github.com/elastic/elasticsearch/pull/122601)) +* Improve error message for ( and [ [#124177](https://github.com/elastic/elasticsearch/pull/124177) (issue: [#124177](https://github.com/elastic/elasticsearch/pull/124177)) * Lazy collection copying during node transform [#124424](https://github.com/elastic/elasticsearch/pull/124424) * Reduce iteration complexity for plan traversal [#123427](https://github.com/elastic/elasticsearch/pull/123427) * Remove duplicated nested commands [#123085](https://github.com/elastic/elasticsearch/pull/123085) * Revive inlinestats [#122257](https://github.com/elastic/elasticsearch/pull/122257) * Revive some more of inlinestats functionality [#123589](https://github.com/elastic/elasticsearch/pull/123589) -* Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: {es-issue}123967[#123967]) +* TO_LOWER processes all values [#124676](https://github.com/elastic/elasticsearch/pull/124676) (issue: [#124676](https://github.com/elastic/elasticsearch/pull/124676)) +* Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: [#124001](https://github.com/elastic/elasticsearch/pull/124001)) Engine: -* Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: {es-issue}122974[#122974]) +* Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: [#123010](https://github.com/elastic/elasticsearch/pull/123010)) + +Highlighting: +* Restore V8 REST compatibility around highlight `force_source` parameter [#124873](https://github.com/elastic/elasticsearch/pull/124873) Indices APIs: -* Avoid hoarding cluster state references during rollover [#124107](https://github.com/elastic/elasticsearch/pull/124107) (issue: {es-issue}123893[#123893]) +* Avoid hoarding cluster state references during rollover [#124107](https://github.com/elastic/elasticsearch/pull/124107) (issue: [#124107](https://github.com/elastic/elasticsearch/pull/124107)) * Updates the deprecation info API to not warn about system indices and data streams [#122951](https://github.com/elastic/elasticsearch/pull/122951) Infra/Core: +* Fix system data streams to be restorable from a snapshot [#124651](https://github.com/elastic/elasticsearch/pull/124651) (issue: [#124651](https://github.com/elastic/elasticsearch/pull/124651)) * Have create index return a bad request on poor formatting [#123761](https://github.com/elastic/elasticsearch/pull/123761) * Include data streams when converting an existing resource to a system resource [#121392](https://github.com/elastic/elasticsearch/pull/121392) +* Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. [#124732](https://github.com/elastic/elasticsearch/pull/124732) (issue: [#124732](https://github.com/elastic/elasticsearch/pull/124732)) * Reduce Data Loss in System Indices Migration [#121327](https://github.com/elastic/elasticsearch/pull/121327) * System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) @@ -182,8 +256,10 @@ Machine Learning: * Add `ElasticInferenceServiceCompletionServiceSettings` [#123155](https://github.com/elastic/elasticsearch/pull/123155) * Add enterprise license check to inference action for semantic text fields [#122293](https://github.com/elastic/elasticsearch/pull/122293) * Avoid potentially throwing calls to Task#getDescription in model download [#124527](https://github.com/elastic/elasticsearch/pull/124527) +* Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` [#125023](https://github.com/elastic/elasticsearch/pull/125023) * Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) -* Provide model size statistics as soon as an anomaly detection job is opened [#124638](https://github.com/elastic/elasticsearch/pull/124638) (issue: {es-issue}121168[#121168]) +* Migrate `model_version` to `model_id` when parsing persistent elser inference endpoints [#124769](https://github.com/elastic/elasticsearch/pull/124769) (issue: [#124769](https://github.com/elastic/elasticsearch/pull/124769)) +* Provide model size statistics as soon as an anomaly detection job is opened [#124638](https://github.com/elastic/elasticsearch/pull/124638) (issue: [#124638](https://github.com/elastic/elasticsearch/pull/124638)) * Retry on streaming errors [#123076](https://github.com/elastic/elasticsearch/pull/123076) * Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) * Updates to allow using Cohere binary embedding response in semantic search queries [#121827](https://github.com/elastic/elasticsearch/pull/121827) @@ -191,8 +267,10 @@ Machine Learning: Mapping: * Avoid serializing empty `_source` fields in mappings [#122606](https://github.com/elastic/elasticsearch/pull/122606) +* Merge template mappings properly during validation [#124784](https://github.com/elastic/elasticsearch/pull/124784) (issue: [#124784](https://github.com/elastic/elasticsearch/pull/124784)) Ranking: +* Fix LTR query feature with phrases (and two-phase) queries [#125103](https://github.com/elastic/elasticsearch/pull/125103) * Restore `TextSimilarityRankBuilder` XContent output [#124564](https://github.com/elastic/elasticsearch/pull/124564) Relevance: @@ -200,23 +278,25 @@ Relevance: Search: * Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions [#123890](https://github.com/elastic/elasticsearch/pull/123890) +* Don't generate stacktrace in `TaskCancelledException` [#125002](https://github.com/elastic/elasticsearch/pull/125002) * Fix concurrency issue in `ScriptSortBuilder` [#123757](https://github.com/elastic/elasticsearch/pull/123757) * Fix handling of auto expand replicas for stateless indices [#122365](https://github.com/elastic/elasticsearch/pull/122365) -* Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: {es-issue}122186[#122186]) -* Let MLTQuery throw IAE when no analyzer is set [#124662](https://github.com/elastic/elasticsearch/pull/124662) (issue: {es-issue}124562[#124562]) +* Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: [#122357](https://github.com/elastic/elasticsearch/pull/122357)) +* Let MLTQuery throw IAE when no analyzer is set [#124662](https://github.com/elastic/elasticsearch/pull/124662) (issue: [#124662](https://github.com/elastic/elasticsearch/pull/124662)) Snapshot/Restore: * Fork post-snapshot-delete cleanup off master thread [#122731](https://github.com/elastic/elasticsearch/pull/122731) -* Limit number of suppressed S3 deletion errors [#123630](https://github.com/elastic/elasticsearch/pull/123630) (issue: {es-issue}123354[#123354]) +* Limit number of suppressed S3 deletion errors [#123630](https://github.com/elastic/elasticsearch/pull/123630) (issue: [#123630](https://github.com/elastic/elasticsearch/pull/123630)) +* This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. [#124931](https://github.com/elastic/elasticsearch/pull/124931) Suggesters: -* Return an empty suggestion when suggest phase times out [#122575](https://github.com/elastic/elasticsearch/pull/122575) (issue: {es-issue}122548[#122548]) -* Support duplicate suggestions in completion field [#121324](https://github.com/elastic/elasticsearch/pull/121324) (issue: {es-issue}82432[#82432]) +* Return an empty suggestion when suggest phase times out [#122575](https://github.com/elastic/elasticsearch/pull/122575) (issue: [#122575](https://github.com/elastic/elasticsearch/pull/122575)) +* Support duplicate suggestions in completion field [#121324](https://github.com/elastic/elasticsearch/pull/121324) (issue: [#121324](https://github.com/elastic/elasticsearch/pull/121324)) Transform: -* If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias [#122074](https://github.com/elastic/elasticsearch/pull/122074) (issue: {es-issue}121913[#121913]) +* If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias [#122074](https://github.com/elastic/elasticsearch/pull/122074) (issue: [#122074](https://github.com/elastic/elasticsearch/pull/122074)) Vector Search: -* Knn vector rescoring to sort score docs [#122653](https://github.com/elastic/elasticsearch/pull/122653) (issue: {es-issue}119711[#119711]) +* Knn vector rescoring to sort score docs [#122653](https://github.com/elastic/elasticsearch/pull/122653) (issue: [#122653](https://github.com/elastic/elasticsearch/pull/122653)) From 056529543edda7e4b55c69d81ea56b113c470cba Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 21 Mar 2025 11:35:53 -0400 Subject: [PATCH 10/34] Remove now unused release notes stuff --- .../release/ReleaseHighlightsGenerator.java | 68 --------------- .../release/ReleaseNotesIndexGenerator.java | 54 ------------ .../ReleaseHighlightsGeneratorTest.java | 87 ------------------- 3 files changed, 209 deletions(-) delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java delete mode 100644 build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java deleted file mode 100644 index 25e4bec79c353..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import com.google.common.annotations.VisibleForTesting; - -import org.elasticsearch.gradle.VersionProperties; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * Generates the release highlights notes, for changelog files that contain the highlight field. - */ -public class ReleaseHighlightsGenerator { - static void update(File templateFile, File outputFile, List entries) throws IOException { - try (FileWriter output = new FileWriter(outputFile)) { - output.write( - generateFile(QualifiedVersion.of(VersionProperties.getElasticsearch()), Files.readString(templateFile.toPath()), entries) - ); - } - } - - @VisibleForTesting - static String generateFile(QualifiedVersion version, String template, List entries) throws IOException { - final List priorVersions = new ArrayList<>(); - - if (version.minor() > 0) { - final int major = version.major(); - for (int minor = version.minor() - 1; minor >= 0; minor--) { - String majorMinor = major + "." + minor; - priorVersions.add("{ref-bare}/" + majorMinor + "/release-highlights.html[" + majorMinor + "]"); - } - } - - final Map> groupedHighlights = entries.stream() - .map(ChangelogEntry::getHighlight) - .filter(Objects::nonNull) - .sorted(Comparator.comparingInt(ChangelogEntry.Highlight::getPr)) - .collect(Collectors.groupingBy(ChangelogEntry.Highlight::isNotable, Collectors.toList())); - - final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); - final List nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); - - final Map bindings = new HashMap<>(); - bindings.put("priorVersions", priorVersions); - bindings.put("notableHighlights", notableHighlights); - bindings.put("nonNotableHighlights", nonNotableHighlights); - - return TemplateUtils.render(template, bindings); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java deleted file mode 100644 index b8c195f47d824..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import com.google.common.annotations.VisibleForTesting; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; - -import static java.util.Comparator.reverseOrder; - -/** - * This class ensures that the release notes index page has the appropriate anchors and include directives - * for the current repository version. - */ -public class ReleaseNotesIndexGenerator { - - static void update(Set versions, File indexTemplate, File indexFile) throws IOException { - try (FileWriter indexFileWriter = new FileWriter(indexFile)) { - indexFileWriter.write(generateFile(versions, Files.readString(indexTemplate.toPath()))); - } - } - - @VisibleForTesting - static String generateFile(Set versionsSet, String template) throws IOException { - final Set versions = new TreeSet<>(reverseOrder()); - - // For the purpose of generating the index, snapshot versions are the same as released versions. Prerelease versions are not. - versionsSet.stream().map(v -> v.isSnapshot() ? v.withoutQualifier() : v).forEach(versions::add); - - final List includeVersions = versions.stream().map(QualifiedVersion::toString).collect(Collectors.toList()); - - final Map bindings = new HashMap<>(); - bindings.put("versions", versions); - bindings.put("includeVersions", includeVersions); - - return TemplateUtils.render(template, bindings); - } -} diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java deleted file mode 100644 index 4e8320d8c0b65..0000000000000 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.release; - -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import java.util.Objects; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; - -public class ReleaseHighlightsGeneratorTest { - - /** - * Check that the release highlights can be correctly generated when there are no highlights. - */ - @Test - public void generateFile_withNoHighlights_rendersCorrectMarkup() throws Exception { - // given: - final String template = getResource("/templates/release-highlights.asciidoc"); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.noHighlights.generateFile.asciidoc" - ); - - // when: - final String actualOutput = ReleaseHighlightsGenerator.generateFile(QualifiedVersion.of("8.4.0-SNAPSHOT"), template, List.of()); - - // then: - assertThat(actualOutput, equalTo(expectedOutput)); - } - - /** - * Check that the release highlights can be correctly generated. - */ - @Test - public void generateFile_rendersCorrectMarkup() throws Exception { - // given: - final String template = getResource("/templates/release-highlights.asciidoc"); - final String expectedOutput = getResource( - "/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc" - ); - - final List entries = getEntries(); - - // when: - final String actualOutput = ReleaseHighlightsGenerator.generateFile(QualifiedVersion.of("8.4.0-SNAPSHOT"), template, entries); - - // then: - assertThat(actualOutput, equalTo(expectedOutput)); - } - - private List getEntries() { - ChangelogEntry entry123 = makeChangelogEntry(123, true); - ChangelogEntry entry456 = makeChangelogEntry(456, true); - ChangelogEntry entry789 = makeChangelogEntry(789, false); - // Return unordered list, to test correct re-ordering - return List.of(entry456, entry123, entry789); - } - - private ChangelogEntry makeChangelogEntry(int pr, boolean notable) { - ChangelogEntry entry = new ChangelogEntry(); - entry.setPr(pr); - ChangelogEntry.Highlight highlight = new ChangelogEntry.Highlight(); - entry.setHighlight(highlight); - - highlight.setNotable(notable); - highlight.setTitle("Notable release highlight number " + pr); - highlight.setBody("Notable release body number " + pr); - - return entry; - } - - private String getResource(String name) throws Exception { - return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8); - } -} From 3ccb2f978832aed289d7e8714653f1715e31965a Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 1 Apr 2025 12:39:27 -0400 Subject: [PATCH 11/34] WIP changelog bundles for release notes --- .../release/BundleChangelogsTask.java | 133 ++ .../internal/release/ChangelogBundle.java | 39 + .../internal/release/ChangelogEntry.java | 9 +- .../release/GenerateReleaseNotesTask.java | 170 +-- .../gradle/internal/release/GitWrapper.java | 11 + .../release/ReleaseNotesGenerator.java | 81 +- .../internal/release/ReleaseToolsPlugin.java | 16 +- .../src/main/resources/templates/index.md | 16 +- .../release-notes/changelog-bundles/9.0.0.yml | 1102 +++++++++++++++++ .../release-notes/changelog-bundles/9.1.0.yml | 1097 ++++++++++++++++ docs/release-notes/index.md | 109 +- 11 files changed, 2628 insertions(+), 155 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java create mode 100644 docs/release-notes/changelog-bundles/9.0.0.yml create mode 100644 docs/release-notes/changelog-bundles/9.1.0.yml diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java new file mode 100644 index 0000000000000..77b41adf7d2a1 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.release; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import org.gradle.api.DefaultTask; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.Directory; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.RegularFile; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; +import org.gradle.process.ExecOperations; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Properties; + +import javax.inject.Inject; + +import static java.util.stream.Collectors.toList; + +public class BundleChangelogsTask extends DefaultTask { + private static final Logger LOGGER = Logging.getLogger(BundleChangelogsTask.class); + + private final ConfigurableFileCollection changelogs; + + private final RegularFileProperty bundleFile; + private final DirectoryProperty changelogDirectory; + + private final GitWrapper gitWrapper; + + private static final ObjectMapper yamlMapper = new ObjectMapper( + new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) + .disable(YAMLGenerator.Feature.SPLIT_LINES) + .enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR) + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE) + ).setSerializationInclusion(JsonInclude.Include.NON_NULL); + + @Inject + public BundleChangelogsTask(ObjectFactory objectFactory, ExecOperations execOperations) { + changelogs = objectFactory.fileCollection(); + + bundleFile = objectFactory.fileProperty(); + changelogDirectory = objectFactory.directoryProperty(); + + gitWrapper = new GitWrapper(execOperations); + } + + @TaskAction + public void executeTask() throws IOException { + final String upstreamRemote = gitWrapper.getUpstream(); + + String ref = "main"; + try { + checkoutChangelogs(gitWrapper, upstreamRemote, ref); + Properties props = new Properties(); + props.load(new StringReader(gitWrapper.runCommand("git", "show", ref + ":build-tools-internal/version.properties"))); + String version = props.getProperty("elasticsearch"); + + LOGGER.info("Finding changelog files..."); + + List entries = this.changelogDirectory.getAsFileTree() + .getFiles() + .stream() + .map(ChangelogEntry::parse) + .sorted(Comparator.comparing(ChangelogEntry::getPr)) + .collect(toList()); + + ChangelogBundle bundle = new ChangelogBundle(version, Instant.now().toString(), entries); + + yamlMapper.writeValue(new File("docs/release-notes/changelog-bundles/" + version + ".yml"), bundle); + } finally { + gitWrapper.runCommand("git", "restore", "-s@", "-SW", "--", "docs/changelog"); + } + } + + private static void checkoutChangelogs(GitWrapper gitWrapper, String upstream, String ref) { + gitWrapper.updateRemote(upstream); + // TODO check for changes first + gitWrapper.runCommand("git", "checkout", ref, "--", "docs/changelog"); + } + + @InputDirectory + public DirectoryProperty getChangelogDirectory() { + return changelogDirectory; + } + + public void setChangelogDirectory(Directory dir) { + this.changelogDirectory.set(dir); + } + + @InputFiles + public FileCollection getChangelogs() { + return changelogs; + } + + public void setChangelogs(FileCollection files) { + this.changelogs.setFrom(files); + } + + @OutputFile + public RegularFileProperty getBundleFile() { + return bundleFile; + } + + public void setBundleFile(RegularFile file) { + this.bundleFile.set(file); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java new file mode 100644 index 0000000000000..b458770109353 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.release; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; + +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; + +public record ChangelogBundle(String version, String generated, List changelogs) { + + private static final Logger LOGGER = Logging.getLogger(GenerateReleaseNotesTask.class); + private static final ObjectMapper yamlMapper = new ObjectMapper( + new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES).disable(YAMLGenerator.Feature.SPLIT_LINES) + ); + + public static ChangelogBundle parse(File file) { + try { + return yamlMapper.readValue(file, ChangelogBundle.class); + } catch (IOException e) { + LOGGER.error("Failed to parse changelog bundle from " + file.getAbsolutePath(), e); + throw new UncheckedIOException(e); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java index ff3ad2f944c52..e7b8cad1c8e49 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.release; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -35,12 +36,12 @@ public class ChangelogEntry { private static final Logger LOGGER = Logging.getLogger(GenerateReleaseNotesTask.class); private Integer pr; - private List issues; + private String summary; private String area; private String type; - private String summary; - private Highlight highlight; + private List issues; private Breaking breaking; + private Highlight highlight; private Deprecation deprecation; private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); @@ -193,6 +194,7 @@ public void setBody(String body) { this.body = body; } + @JsonIgnore public String getAnchor() { return generatedAnchor(this.title); } @@ -278,6 +280,7 @@ public void setNotable(boolean notable) { this.notable = notable; } + @JsonIgnore public String getAnchor() { return generatedAnchor(this.title); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index 79e61d833a858..a7aa2375d0a79 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -15,34 +15,31 @@ import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.Directory; +import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.FileCollection; import org.gradle.api.file.RegularFile; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFile; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; import org.gradle.process.ExecOperations; -import java.io.File; import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.Comparator; import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Stream; import javax.inject.Inject; -import static java.util.Comparator.naturalOrder; import static java.util.stream.Collectors.toSet; /** @@ -63,6 +60,8 @@ public class GenerateReleaseNotesTask extends DefaultTask { private final RegularFileProperty breakingChangesFile; private final RegularFileProperty deprecationsFile; + private final DirectoryProperty changelogBundleDirectory; + private final GitWrapper gitWrapper; @Inject @@ -79,6 +78,8 @@ public GenerateReleaseNotesTask(ObjectFactory objectFactory, ExecOperations exec breakingChangesFile = objectFactory.fileProperty(); deprecationsFile = objectFactory.fileProperty(); + changelogBundleDirectory = objectFactory.directoryProperty(); + gitWrapper = new GitWrapper(execOperations); } @@ -90,51 +91,46 @@ public void executeTask() throws IOException { findAndUpdateUpstreamRemote(gitWrapper); } - LOGGER.info("Finding changelog files..."); - - final Map> filesByVersion = partitionFilesByVersion( - gitWrapper, - currentVersion, - this.changelogs.getFiles() - ); + LOGGER.info("Finding changelog bundles..."); - final List entries = new ArrayList<>(); - final Map> changelogsByVersion = new HashMap<>(); - - filesByVersion.forEach((version, files) -> { - Set entriesForVersion = files.stream().map(ChangelogEntry::parse).collect(toSet()); - entries.addAll(entriesForVersion); - changelogsByVersion.put(version, entriesForVersion); - }); + List bundles = this.changelogBundleDirectory.getAsFileTree() + .getFiles() + .stream() + .map(ChangelogBundle::parse) + .sorted(Comparator.comparing(ChangelogBundle::generated).reversed()) + .toList(); - final Set versions = getVersions(gitWrapper, currentVersion); + // Ensure that each changelog/PR only shows up once, in its earliest release + // TODO: This should only be for unreleased/non-final bundles + var uniquePrs = new HashSet(); + for (int i = bundles.size() - 1; i >= 0; i--) { + var bundle = bundles.get(i); + bundle.changelogs().removeAll(bundle.changelogs().stream().filter(c -> uniquePrs.contains(c.getPr())).toList()); + uniquePrs.addAll(bundle.changelogs().stream().map(ChangelogEntry::getPr).toList()); + } LOGGER.info("Generating release notes..."); - final QualifiedVersion qualifiedVersion = QualifiedVersion.of(currentVersion); - ReleaseNotesGenerator.update( - this.releaseNotesTemplate.get().getAsFile(), - this.releaseNotesFile.get().getAsFile(), - qualifiedVersion, - changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) - ); + ReleaseNotesGenerator.update(this.releaseNotesTemplate.get().getAsFile(), this.releaseNotesFile.get().getAsFile(), bundles); // Only update breaking changes and deprecations for new minors - if (qualifiedVersion.revision() == 0) { - LOGGER.info("Generating breaking changes / deprecations notes..."); - ReleaseNotesGenerator.update( - this.breakingChangesTemplate.get().getAsFile(), - this.breakingChangesFile.get().getAsFile(), - qualifiedVersion, - changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) - ); - - ReleaseNotesGenerator.update( - this.deprecationsTemplate.get().getAsFile(), - this.deprecationsFile.get().getAsFile(), - qualifiedVersion, - changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()) - ); - } + // if (qualifiedVersion.revision() == 0) { + // LOGGER.info("Generating breaking changes / deprecations notes..."); + // ReleaseNotesGenerator.update( + // this.breakingChangesTemplate.get().getAsFile(), + // this.breakingChangesFile.get().getAsFile(), + // qualifiedVersion, + // changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()), + // bundles + // ); + // + // ReleaseNotesGenerator.update( + // this.deprecationsTemplate.get().getAsFile(), + // this.deprecationsFile.get().getAsFile(), + // qualifiedVersion, + // changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()), + // bundles + // ); + // } } /** @@ -170,79 +166,6 @@ static Set getMinorVersions(Set versions) { return versions.stream().map(MinorVersion::of).collect(toSet()); } - /** - * Group a set of files by the version in which they first appeared, up until the supplied version. Any files not - * present in an earlier version are assumed to have been introduced in the specified version. - * - *

This method works by finding all git tags prior to {@param versionString} in the same minor series, and - * examining the git tree for that tag. By doing this over each tag, it is possible to see how the contents - * of the changelog directory changed over time. - * - * @param gitWrapper used to call `git` - * @param versionString the "current" version. Does not require a tag in git. - * @param allFilesInCheckout the files to partition - * @return a mapping from version to the files added in that version. - */ - @VisibleForTesting - static Map> partitionFilesByVersion( - GitWrapper gitWrapper, - String versionString, - Set allFilesInCheckout - ) { - if (needsGitTags(versionString) == false) { - return Map.of(QualifiedVersion.of(versionString), allFilesInCheckout); - } - - QualifiedVersion currentVersion = QualifiedVersion.of(versionString); - - // Find all tags for this minor series, using a wildcard tag pattern. - String tagWildcard = String.format(Locale.ROOT, "v%d.%d*", currentVersion.major(), currentVersion.minor()); - - final List earlierVersions = gitWrapper.listVersions(tagWildcard) - // Only keep earlier versions, and if `currentVersion` is a prerelease, then only prereleases too. - .filter( - each -> each.isBefore(currentVersion) - && (currentVersion.isSnapshot() || (currentVersion.hasQualifier() == each.hasQualifier())) - ) - .sorted(naturalOrder()) - .toList(); - - if (earlierVersions.isEmpty()) { - throw new GradleException("Failed to find git tags prior to [v" + currentVersion + "]"); - } - - Map> partitionedFiles = new HashMap<>(); - - Set mutableAllFilesInCheckout = new HashSet<>(allFilesInCheckout); - - // 1. For each earlier version - earlierVersions.forEach(earlierVersion -> { - // 2. Find all the changelog files it contained - Set filesInTreeForVersion = gitWrapper.listFiles("v" + earlierVersion, "docs/changelog") - .map(line -> Path.of(line).getFileName().toString()) - .collect(toSet()); - - Set filesForVersion = new HashSet<>(); - partitionedFiles.put(earlierVersion, filesForVersion); - - // 3. Find the `File` object for each one - final Iterator filesIterator = mutableAllFilesInCheckout.iterator(); - while (filesIterator.hasNext()) { - File nextFile = filesIterator.next(); - if (filesInTreeForVersion.contains(nextFile.getName())) { - // 4. And remove it so that it is associated with the earlier version - filesForVersion.add(nextFile); - filesIterator.remove(); - } - } - }); - - // 5. Associate whatever is left with the current version. - partitionedFiles.put(currentVersion, mutableAllFilesInCheckout); - - return partitionedFiles; - } - /** * Ensure the upstream git remote is up-to-date. The upstream is whatever git remote references `elastic/elasticsearch`. * @param gitWrapper used to call `git` @@ -300,6 +223,15 @@ public void setChangelogs(FileCollection files) { this.changelogs.setFrom(files); } + @InputDirectory + public DirectoryProperty getChangelogBundleDirectory() { + return changelogBundleDirectory; + } + + public void setChangelogBundleDirectory(Directory dir) { + this.changelogBundleDirectory.set(dir); + } + @InputFile public RegularFileProperty getReleaseNotesTemplate() { return releaseNotesTemplate; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java index f50b3c0890220..6c726d4a16f80 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.release; +import org.gradle.api.GradleException; import org.gradle.process.ExecOperations; import java.io.ByteArrayOutputStream; @@ -87,4 +88,14 @@ public Stream listVersions(String pattern) { public Stream listFiles(String ref, String path) { return runCommand("git", "ls-tree", "--name-only", "-r", ref, path).lines(); } + + public String getUpstream() { + String upstream = listRemotes().entrySet() + .stream() + .filter(entry -> entry.getValue().contains("elastic/elasticsearch")) + .findFirst() + .map(Map.Entry::getKey) + .orElseThrow(() -> new GradleException("Couldn't find a git remote for [elastic/elasticsearch]")); + return upstream; + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java index c9fc64171d09e..82aa678491abb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java @@ -15,6 +15,8 @@ import java.io.FileWriter; import java.io.IOException; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,6 +34,17 @@ * type of change, then by team area. */ public class ReleaseNotesGenerator { + + private record ChangelogsBundleWrapper( + QualifiedVersion version, + ChangelogBundle bundle, + Map>> changelogsByTypeByArea, + QualifiedVersion unqualifiedVersion, + String versionWithoutSeparator, + List notableHighlights, + List nonNotableHighlights + ) {} + /** * These mappings translate change types into the headings as they should appear in the release notes. */ @@ -56,35 +69,67 @@ public class ReleaseNotesGenerator { */ private static final List FEATURE_ENHANCEMENT_TYPES = List.of("feature", "new-aggregation", "enhancement", "upgrade"); - static void update(File templateFile, File outputFile, QualifiedVersion version, Set changelogs) throws IOException { + static void update(File templateFile, File outputFile, List bundles) throws IOException { final String templateString = Files.readString(templateFile.toPath()); try (FileWriter output = new FileWriter(outputFile)) { - output.write(generateFile(templateString, version, changelogs)); + output.write(generateFile(templateString, bundles)); } } @VisibleForTesting - static String generateFile(String template, QualifiedVersion version, Set changelogs) throws IOException { - final var changelogsByTypeByArea = buildChangelogBreakdown(changelogs); + static String generateFile(String template, List bundles) throws IOException { + var bundlesWrapped = new ArrayList(); + + for (var bundle : bundles) { + var changelogs = bundle.changelogs(); + final var changelogsByTypeByArea = buildChangelogBreakdown(changelogs); + + final Map> groupedHighlights = changelogs.stream() + .map(ChangelogEntry::getHighlight) + .filter(Objects::nonNull) + .sorted(comparingInt(ChangelogEntry.Highlight::getPr)) + .collect(groupingBy(ChangelogEntry.Highlight::isNotable, toList())); + + final var notableHighlights = groupedHighlights.getOrDefault(true, List.of()); + final var nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); + + final var version = QualifiedVersion.of(bundle.version()); + final var versionWithoutSeparator = version.withoutQualifier().toString().replaceAll("\\.", ""); + + final var wrapped = new ChangelogsBundleWrapper( + version, + bundle, + changelogsByTypeByArea, + version.withoutQualifier(), + versionWithoutSeparator, + notableHighlights, + nonNotableHighlights + ); - final Map> groupedHighlights = changelogs.stream() - .map(ChangelogEntry::getHighlight) - .filter(Objects::nonNull) - .sorted(comparingInt(ChangelogEntry.Highlight::getPr)) - .collect(groupingBy(ChangelogEntry.Highlight::isNotable, toList())); + bundlesWrapped.add(wrapped); + } - final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); - final List nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); + // final var changelogsByTypeByArea = buildChangelogBreakdown(changelogs); + // + // final Map> groupedHighlights = changelogs.stream() + // .map(ChangelogEntry::getHighlight) + // .filter(Objects::nonNull) + // .sorted(comparingInt(ChangelogEntry.Highlight::getPr)) + // .collect(groupingBy(ChangelogEntry.Highlight::isNotable, toList())); + // + // final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); + // final List nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); final Map bindings = new HashMap<>(); - bindings.put("version", version); - bindings.put("changelogsByTypeByArea", changelogsByTypeByArea); + // bindings.put("version", version); + // bindings.put("changelogsByTypeByArea", changelogsByTypeByArea); bindings.put("TYPE_LABELS", TYPE_LABELS); - bindings.put("unqualifiedVersion", version.withoutQualifier()); - bindings.put("versionWithoutSeparator", version.withoutQualifier().toString().replaceAll("\\.", "")); - bindings.put("notableHighlights", notableHighlights); - bindings.put("nonNotableHighlights", nonNotableHighlights); + // bindings.put("unqualifiedVersion", version.withoutQualifier()); + // bindings.put("versionWithoutSeparator", version.withoutQualifier().toString().replaceAll("\\.", "")); + // bindings.put("notableHighlights", notableHighlights); + // bindings.put("nonNotableHighlights", nonNotableHighlights); + bindings.put("changelogBundles", bundlesWrapped); return TemplateUtils.render(template, bindings); } @@ -109,7 +154,7 @@ private static String getTypeFromEntry(ChangelogEntry entry) { return entry.getType(); } - private static Map>> buildChangelogBreakdown(Set changelogs) { + private static Map>> buildChangelogBreakdown(Collection changelogs) { Map>> changelogsByTypeByArea = changelogs.stream() .collect( groupingBy( diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index f466c4ceeb6a8..db05bb2d32cd1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -55,9 +55,8 @@ public void apply(Project project) { project.getTasks().register("tagVersions", TagVersionsTask.class); project.getTasks().register("setCompatibleVersions", SetCompatibleVersionsTask.class, t -> t.setThisVersion(version)); - final FileTree yamlFiles = projectDirectory.dir("docs/changelog") - .getAsFileTree() - .matching(new PatternSet().include("**/*.yml", "**/*.yaml")); + final Directory changeLogDirectory = projectDirectory.dir("docs/changelog"); + final FileTree yamlFiles = changeLogDirectory.getAsFileTree().matching(new PatternSet().include("**/*.yml", "**/*.yaml")); final Provider validateChangelogsTask = project.getTasks() .register("validateChangelogs", ValidateYamlAgainstSchemaTask.class, task -> { @@ -68,6 +67,14 @@ public void apply(Project project) { task.setReport(new File(project.getBuildDir(), "reports/validateYaml.txt")); }); + final Function> configureBundleTask = shouldConfigureYamlFiles -> task -> { + task.setGroup("Documentation"); + task.setDescription("Generates release notes from changelog files held in this checkout"); + task.setChangelogs(yamlFiles); + task.setChangelogDirectory(changeLogDirectory); + task.setBundleFile(projectDirectory.file("docs/release-notes/changelogs-" + version.toString() + ".yml")); + }; + final Function> configureGenerateTask = shouldConfigureYamlFiles -> task -> { task.setGroup("Documentation"); if (shouldConfigureYamlFiles) { @@ -89,9 +96,12 @@ public void apply(Project project) { task.setDeprecationsTemplate(projectDirectory.file(RESOURCES + "templates/deprecations.md")); task.setDeprecationsFile(projectDirectory.file("docs/release-notes/deprecations.md")); + task.setChangelogBundleDirectory(projectDirectory.dir("docs/release-notes/changelog-bundles")); + task.dependsOn(validateChangelogsTask); }; + project.getTasks().register("bundleChangelogs", BundleChangelogsTask.class).configure(configureBundleTask.apply(true)); project.getTasks().register("generateReleaseNotes", GenerateReleaseNotesTask.class).configure(configureGenerateTask.apply(true)); project.getTasks() .register("generateStubReleaseNotes", GenerateReleaseNotesTask.class) diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md index 02aabcd30d631..dcb87b45c8dfb 100644 --- a/build-tools-internal/src/main/resources/templates/index.md +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -3,8 +3,6 @@ navigation_title: "Elasticsearch" mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-${unqualifiedVersion}.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-${version.major}.${version.minor}.html --- # Elasticsearch release notes [elasticsearch-release-notes] @@ -22,9 +20,16 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * - +<% +for(bundle in changelogBundles) { + def version = bundle.version + def versionWithoutSeparator = bundle.versionWithoutSeparator + def changelogsByTypeByArea = bundle.changelogsByTypeByArea + def notableHighlights = bundle.notableHighlights + def nonNotableHighlights = bundle.nonNotableHighlights + def unqualifiedVersion = bundle.unqualifiedVersion +%> ## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-release-notes] -**Release date:** April 01, 2025 <% if (!notableHighlights.isEmpty() || !nonNotableHighlights.isEmpty()) { print "\n### Highlights [elasticsearch-${versionWithoutSeparator}-highlights]\n" @@ -62,4 +67,5 @@ for (changeType in ['features-enhancements', 'fixes', 'regression']) { } } } -print "\n\n" +print "\n" +} diff --git a/docs/release-notes/changelog-bundles/9.0.0.yml b/docs/release-notes/changelog-bundles/9.0.0.yml new file mode 100644 index 0000000000000..8cfe6dbfc17db --- /dev/null +++ b/docs/release-notes/changelog-bundles/9.0.0.yml @@ -0,0 +1,1102 @@ +version: 9.0.0 +generated: 2025-03-31T18:34:33.370656Z +changelogs: + - pr: 113757 + summary: Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. + area: Mapping + type: enhancement + issues: [] + - pr: 117642 + summary: Adding endpoint creation validation to `ElasticInferenceService` + area: Machine Learning + type: enhancement + issues: [] + - pr: 119503 + summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. + area: Search + type: bug + issues: + - 117042 + - pr: 119546 + summary: Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields + area: Mapping + type: enhancement + issues: [] + - pr: 119886 + summary: Initial support for unmapped fields + area: ES|QL + type: feature + issues: [] + - pr: 119995 + summary: "apm-data: Use representative count as event.success_count if available" + area: Ingest Node + type: bug + issues: [] + - pr: 120751 + summary: Adding support for binary embedding type to Cohere service embedding type + area: Machine Learning + type: enhancement + issues: [] + - pr: 120774 + summary: Retry ES|QL node requests on shard level failures + area: ES|QL + type: enhancement + issues: [] + - pr: 120869 + summary: Threadpool merge scheduler + area: Engine + type: feature + issues: [] + - pr: 120957 + summary: Introduce `AllocationBalancingRoundSummaryService` + area: Allocation + type: enhancement + issues: [] + - pr: 120998 + summary: ES|QL `change_point` processing command + area: Machine Learning + type: feature + issues: [] + - pr: 121106 + summary: Add `ModelRegistryMetadata` to Cluster State + area: Machine Learning + type: enhancement + issues: [] + - pr: 121240 + summary: Implement runtime skip_unavailable=true + area: ES|QL + type: enhancement + issues: [] + - pr: 121256 + summary: Run `TransportEnrichStatsAction` on local node + area: Ingest Node + type: enhancement + issues: [] + - pr: 121260 + summary: Introduce a pre-mapping logical plan processing step + area: ES|QL + type: enhancement + issues: [] + - pr: 121324 + summary: Support duplicate suggestions in completion field + area: Suggesters + type: bug + issues: + - 82432 + - pr: 121327 + summary: Reduce Data Loss in System Indices Migration + area: Infra/Core + type: bug + issues: [] + - pr: 121370 + summary: Improve SLM Health Indicator to cover missing snapshot + area: ILM+SLM + type: enhancement + issues: [] + - pr: 121392 + summary: Include data streams when converting an existing resource to a system resource + area: Infra/Core + type: bug + issues: [] + - pr: 121548 + summary: Adding support for specifying embedding type to Jina AI service settings + area: Machine Learning + type: enhancement + issues: [] + - pr: 121667 + summary: Add deprecation warning for flush API + area: Machine Learning + type: deprecation + issues: + - 121506 + deprecation: + area: REST API + title: Add deprecation warning for flush API + details: "The anomaly detection job flush API is deprecated since it is only required for the post data API, which was deprecated since 7.11.0." + impact: "This should have a minimal impact on users as the flush API is only required for the post data API, which was deprecated since 7.11.0." + notable: false + essSettingChange: false + - pr: 121784 + summary: Optionally allow text similarity reranking to fail + area: Search + type: enhancement + issues: [] + - pr: 121787 + summary: Added optional parameters to QSTR ES|QL function + area: Search + type: feature + issues: + - 120933 + - pr: 121793 + summary: ES|QL - Add scoring for full text functions disjunctions + area: ES|QL + type: enhancement + issues: [] + - pr: 121805 + summary: Support subset of metrics in aggregate metric double + area: ES|QL + type: enhancement + issues: [] + - pr: 121827 + summary: Updates to allow using Cohere binary embedding response in semantic search queries + area: Machine Learning + type: bug + issues: [] + - pr: 121920 + summary: Account for the `SearchHit` source in circuit breaker + area: Search + type: enhancement + issues: + - 89656 + - pr: 121942 + summary: Allow partial results in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 121948 + summary: Add initial grammar and changes for FORK + area: ES|QL + type: feature + issues: [] + - pr: 121971 + summary: Do not fetch reserved roles from native store when Get Role API is called + area: Authorization + type: enhancement + issues: [] + - pr: 122062 + summary: Upgrade `discovery-ec2` to AWS SDK v2 + area: Discovery-Plugins + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Upgrade `discovery-ec2` to AWS SDK v2 + details: |2- + + In earlier versions of {es} the `discovery-ec2` plugin was based on the AWS SDK v1. AWS will withdraw support for this SDK before the end of the life of {es} {minor-version} so we must migrate to the newer AWS SDK v2. + Unfortunately there are several differences between the two AWS SDK versions which may require you to adjust your system configuration when upgrading to {es} {minor-version} or later. These differences include, but may not be limited to, the following items. + * AWS SDK v2 does not support the EC2 IMDSv1 protocol. + * AWS SDK v2 does not support the `aws.secretKey` or + `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties. + + * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so + the `discovery.ec2.protocol` setting is no longer effective. + + * AWS SDK v2 does not accept an access key without a secret key or vice + versa. + impact: |2- + + If you use the `discovery-ec2` plugin, test your upgrade thoroughly before upgrading any production workloads. + Adapt your configuration to the new SDK functionality. This includes, but may not be limited to, the following items. + * If you use IMDS to determine the availability zone of a node or to obtain + credentials for accessing the EC2 API, ensure that it supports the IMDSv2 + protocol. + + * If applicable, discontinue use of the `aws.secretKey` and + `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties. + + * If applicable, specify that you wish to use the insecure HTTP protocol to + access the EC2 API by setting `discovery.ec2.endpoint` to a URL which + starts with `http://`. + + * Either supply both an access key and a secret key using the keystore + settings `discovery.ec2.access_key` and `discovery.ec2.secret_key`, or + configure neither of these settings. + notable: true + essSettingChange: false + - pr: 122066 + summary: Adding elser default endpoint for EIS + area: Machine Learning + type: enhancement + issues: [] + - pr: 122074 + summary: "If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias" + area: Transform + type: bug + issues: + - 121913 + - pr: 122134 + summary: Adding integration for VoyageAI embeddings and rerank models + area: Machine Learning + type: enhancement + issues: [] + - pr: 122218 + summary: Integrate with `DeepSeek` API + area: Machine Learning + type: enhancement + issues: [] + - pr: 122224 + summary: Enable the use of nested field type with index.mode=time_series + area: Mapping + type: enhancement + issues: + - 120874 + - pr: 122250 + summary: "ESQL: Align `RENAME` behavior with `EVAL` for sequential processing" + area: ES|QL + type: enhancement + issues: + - 121739 + - pr: 122257 + summary: Revive inlinestats + area: ES|QL + type: bug + issues: [] + - pr: 122272 + summary: "[Inference API] Rename `model_id` prop to model in EIS sparse inference request body" + area: Inference + type: enhancement + issues: [] + - pr: 122278 + summary: Fix serialising the inference update request + area: Machine Learning + type: bug + issues: [] + - pr: 122280 + summary: Use `FallbackSyntheticSourceBlockLoader` for number fields + area: Mapping + type: enhancement + issues: [] + - pr: 122293 + summary: Add enterprise license check to inference action for semantic text fields + area: Machine Learning + type: bug + issues: [] + - pr: 122326 + summary: System Index Migration Failure Results in a Non-Recoverable State + area: Infra/Core + type: bug + issues: [] + - pr: 122357 + summary: Handle search timeout in `SuggestPhase` + area: Search + type: bug + issues: + - 122186 + - pr: 122365 + summary: Fix handling of auto expand replicas for stateless indices + area: Search + type: bug + issues: [] + - pr: 122381 + summary: Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields + area: Vector Search + type: enhancement + issues: + - 117274 + - pr: 122390 + summary: Add health indicator impact to `HealthPeriodicLogger` + area: Health + type: enhancement + issues: [] + - pr: 122409 + summary: Allow setting the `type` in the reroute processor + area: Ingest Node + type: enhancement + issues: + - 121553 + - pr: 122427 + summary: Improve size limiting string message + area: Infra/Core + type: enhancement + issues: [] + - pr: 122431 + summary: Upgrade AWS SDK to v1.12.746 + area: Snapshot/Restore + type: upgrade + issues: [] + - pr: 122458 + summary: '`DesiredBalanceReconciler` always returns `AllocationStats`' + area: Allocation + type: bug + issues: [] + - pr: 122459 + summary: Double parameter markers for identifiers + area: ES|QL + type: enhancement + issues: [] + - pr: 122486 + summary: Add index mode to get data stream API + area: Data streams + type: enhancement + issues: [] + - pr: 122575 + summary: Return an empty suggestion when suggest phase times out + area: Suggesters + type: bug + issues: + - 122548 + - pr: 122586 + summary: "ESQL: Fix inconsistent results in using scaled_float field" + area: ES|QL + type: bug + issues: + - 122547 + - pr: 122601 + summary: Implicit numeric casting for CASE/GREATEST/LEAST + area: ES|QL + type: bug + issues: + - 121890 + - pr: 122606 + summary: Avoid serializing empty `_source` fields in mappings + area: Mapping + type: bug + issues: [] + - pr: 122615 + summary: Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. + area: Mapping + type: enhancement + issues: + - 116726 + - pr: 122637 + summary: Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields + area: Mapping + type: enhancement + issues: [] + - pr: 122638 + summary: Expose `input_type` option at root level for `text_embedding` task type in Perform Inference API + area: Machine Learning + type: enhancement + issues: + - 117856 + - pr: 122653 + summary: Knn vector rescoring to sort score docs + area: Vector Search + type: bug + issues: + - 119711 + - pr: 122660 + summary: Render `aggregate_metric_double` + area: ES|QL + type: enhancement + issues: [] + - pr: 122708 + summary: Support partial results in CCS in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 122731 + summary: Fork post-snapshot-delete cleanup off master thread + area: Snapshot/Restore + type: bug + issues: [] + - pr: 122762 + summary: "ESQL: Remove estimated row size assertion" + area: ES|QL + type: bug + issues: + - 121535 + - pr: 122821 + summary: Fix functions emitting warnings with no source + area: ES|QL + type: bug + issues: + - 122588 + - pr: 122823 + summary: Prevent Query Rule Creation with Invalid Numeric Match Criteria + area: Relevance + type: bug + issues: [] + - pr: 122852 + summary: Run `TransportGetDataStreamsAction` on local node + area: Data streams + type: enhancement + issues: [] + - pr: 122857 + summary: Run `TransportGetWatcherSettingsAction` on local node + area: Watcher + type: enhancement + issues: [] + - pr: 122860 + summary: Improved error message when index field type is invalid + area: Mapping + type: enhancement + issues: [] + - pr: 122885 + summary: Run `TransportExplainLifecycleAction` on local node + area: ILM+SLM + type: enhancement + issues: [] + - pr: 122886 + summary: Add support to VALUES aggregation for spatial types + area: ES|QL + type: bug + issues: + - 122413 + - pr: 122890 + summary: Introduce `allow_partial_results` setting in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 122891 + summary: Pragma to load from stored fields + area: ES|QL + type: enhancement + issues: [] + - pr: 122905 + summary: Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover + area: Data streams + type: bug + issues: [] + - pr: 122921 + summary: Run `TransportGetMappingsAction` on local node + area: Indices APIs + type: enhancement + issues: [] + - pr: 122933 + summary: Run XPack usage actions on local node + area: Stats + type: enhancement + issues: [] + - pr: 122938 + summary: Fix geoip databases index access after system feature migration (again) + area: Ingest Node + type: bug + issues: [] + - pr: 122951 + summary: Updates the deprecation info API to not warn about system indices and data streams + area: Indices APIs + type: bug + issues: [] + - pr: 122960 + summary: Deprecate Behavioral Analytics CRUD apis + area: Search + type: deprecation + issues: [] + deprecation: + area: Search + title: Deprecate Behavioral Analytics CRUD apis + details: "Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. The APIs will still work for now, but will emit warning headers that the API has been deprecated." + impact: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. + notable: false + essSettingChange: false + - pr: 122991 + summary: "GCS blob store: add `OperationPurpose/Operation` stats counters" + area: Snapshot/Restore + type: enhancement + issues: [] + - pr: 122999 + summary: Store arrays offsets for ip fields natively with synthetic source + area: Mapping + type: enhancement + issues: [] + - pr: 123010 + summary: Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` + area: Engine + type: bug + issues: + - 122974 + - pr: 123076 + summary: Retry on streaming errors + area: Machine Learning + type: bug + issues: [] + - pr: 123085 + summary: Remove duplicated nested commands + area: ES|QL + type: bug + issues: [] + - pr: 123150 + summary: Limit the number of chunks for semantic text to prevent high memory usage + area: Machine Learning + type: feature + issues: [] + - pr: 123155 + summary: Add `ElasticInferenceServiceCompletionServiceSettings` + area: Machine Learning + type: bug + issues: [] + - pr: 123187 + summary: Add bit vector support to semantic text + area: Vector Search + type: enhancement + issues: [] + - pr: 123197 + summary: Fix early termination in `LuceneSourceOperator` + area: ES|QL + type: bug + issues: [] + - pr: 123272 + summary: Set Connect Timeout to 5s + area: Machine Learning + type: bug + issues: [] + - pr: 123290 + summary: Fix Driver status iterations and `cpuTime` + area: ES|QL + type: enhancement + issues: + - 122967 + - pr: 123296 + summary: Avoid over collecting in Limit or Lucene Operator + area: ES|QL + type: bug + issues: [] + - pr: 123381 + summary: Push down `StartsWith` and `EndsWith` functions to Lucene + area: ES|QL + type: enhancement + issues: + - 123067 + - pr: 123396 + summary: Add initial grammar and planning for RRF (snapshot) + area: ES|QL + type: feature + issues: [] + - pr: 123427 + summary: Reduce iteration complexity for plan traversal + area: ES|QL + type: bug + issues: [] + - pr: 123460 + summary: "ES|QL: Support `::date` in inline cast" + area: ES|QL + type: enhancement + issues: + - 116746 + - pr: 123492 + summary: Fix function registry concurrency issues on constructor + area: ES|QL + type: bug + issues: + - 123430 + - pr: 123543 + summary: Enhance memory accounting for document expansion and introduce max document size limit + area: CRUD + type: enhancement + issues: [] + - pr: 123569 + summary: Abort pending deletion on `IndicesService` close + area: Store + type: enhancement + issues: [] + - pr: 123588 + summary: Give Kibana user 'all' permissions for .entity_analytics.* indices + area: Infra/Core + type: enhancement + issues: [] + - pr: 123589 + summary: Revive some more of inlinestats functionality + area: ES|QL + type: bug + issues: [] + - pr: 123600 + summary: Drop `TLS_RSA` cipher support for JDK 24 + area: TLS + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Drop `TLS_RSA` cipher support for JDK 24 + details: "This change removes `TLS_RSA` ciphers from the list of default supported ciphers, for Elasticsearch deployments running on JDK 24." + impact: "The dropped ciphers are `TLS_RSA_WITH_AES_256_GCM_SHA384`, `TLS_RSA_WITH_AES_128_GCM_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA`, and `TLS_RSA_WITH_AES_128_CBC_SHA`. TLS connections to Elasticsearch using these ciphers will no longer work. Please configure your clients to use one of supported cipher suites." + notable: false + essSettingChange: false + - pr: 123630 + summary: Limit number of suppressed S3 deletion errors + area: Snapshot/Restore + type: bug + issues: + - 123354 + - pr: 123712 + summary: Process ILM cluster state updates on another thread + area: ILM+SLM + type: enhancement + issues: [] + - pr: 123728 + summary: Allow skip shards with `_tier` and `_index` in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 123743 + summary: Adjust exception thrown when unable to load hunspell dict + area: Analysis + type: bug + issues: [] + - pr: 123757 + summary: Fix concurrency issue in `ScriptSortBuilder` + area: Search + type: bug + issues: [] + - pr: 123761 + summary: Have create index return a bad request on poor formatting + area: Infra/Core + type: bug + issues: [] + - pr: 123763 + summary: Skip semantic_text embedding generation when no content is provided. + area: Relevance + type: enhancement + issues: [] + - pr: 123852 + summary: Retry when the server can't be resolved (Google Cloud Storage) + area: Snapshot/Restore + type: enhancement + issues: [] + - pr: 123890 + summary: Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions + area: Search + type: bug + issues: [] + - pr: 124001 + summary: Use a must boolean statement when pushing down to Lucene when scoring is also needed + area: ES|QL + type: bug + issues: + - 123967 + - pr: 124025 + summary: "[Inference API] Propagate product use case http header to EIS" + area: Machine Learning + type: enhancement + issues: [] + - pr: 124048 + summary: Handle long overflow in dates + area: Search + type: bug + issues: + - 112483 + - pr: 124050 + summary: Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields + area: Mapping + type: enhancement + issues: [] + - pr: 124062 + summary: Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0 + area: Snapshot/Restore + type: upgrade + issues: [] + - pr: 124094 + summary: ES|QL slow log + area: ES|QL + type: enhancement + issues: [] + - pr: 124107 + summary: Avoid hoarding cluster state references during rollover + area: Indices APIs + type: bug + issues: + - 123893 + - pr: 124149 + summary: Retry ILM async action after reindexing data stream + area: Data streams + type: enhancement + issues: [] + - pr: 124177 + summary: "Improve error message for ( and [" + area: ES|QL + type: bug + issues: + - 124145 + - pr: 124225 + summary: "[Inference API] Fix output stream ordering in `InferenceActionProxy`" + area: Machine Learning + type: bug + issues: [] + - pr: 124313 + summary: Optimize memory usage in `ShardBulkInferenceActionFilter` + area: Search + type: enhancement + issues: [] + - pr: 124335 + summary: Change the order of the optimization rules + area: ES|QL + type: bug + issues: [] + - pr: 124363 + summary: Set cause on create index request in create from action + area: Data streams + type: enhancement + issues: [] + - pr: 124394 + summary: Avoid `NamedWritable` in block serialization + area: ES|QL + type: enhancement + issues: [] + - pr: 124424 + summary: Lazy collection copying during node transform + area: ES|QL + type: bug + issues: [] + - pr: 124446 + summary: "ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate`" + area: ES|QL + type: bug + issues: + - 124311 + - pr: 124451 + summary: Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. + area: Downsampling + type: bug + issues: [] + - pr: 124477 + summary: Improve downsample performance by buffering docids and do bulk processing + area: Downsampling + type: enhancement + issues: [] + - pr: 124527 + summary: Avoid potentially throwing calls to Task#getDescription in model download + area: Machine Learning + type: bug + issues: [] + - pr: 124540 + summary: "ES|QL: Fix scoring for full text functions" + area: ES|QL + type: bug + issues: [] + - pr: 124544 + summary: Bump nimbus-jose-jwt to 10.0.2 + area: Security + type: upgrade + issues: [] + - pr: 124564 + summary: Restore `TextSimilarityRankBuilder` XContent output + area: Ranking + type: bug + issues: [] + - pr: 124574 + summary: Allow passing several reserved state chunks in single process call + area: Infra/Settings + type: enhancement + issues: [] + - pr: 124581 + summary: New `vector_rescore` parameter as a quantized index type option + area: Vector Search + type: enhancement + issues: [] + - pr: 124594 + summary: Store arrays offsets for numeric fields natively with synthetic source + area: Mapping + type: enhancement + issues: [] + - pr: 124595 + summary: '`ToAggregateMetricDouble` function' + area: ES|QL + type: enhancement + issues: [] + - pr: 124604 + summary: Fix geoip databases index access after system feature migration (take 3) + area: Ingest Node + type: bug + issues: [] + - pr: 124610 + summary: Remove page alignment in exchange sink + area: ES|QL + type: enhancement + issues: [] + - pr: 124611 + summary: Reuse child `outputSet` inside the plan where possible + area: ES|QL + type: enhancement + issues: [] + - pr: 124638 + summary: Provide model size statistics as soon as an anomaly detection job is opened + area: Machine Learning + type: bug + issues: + - 121168 + - pr: 124651 + summary: Fix system data streams to be restorable from a snapshot + area: Infra/Core + type: bug + issues: + - 89261 + - pr: 124662 + summary: Let MLTQuery throw IAE when no analyzer is set + area: Search + type: bug + issues: + - 124562 + - pr: 124669 + summary: Release semantic_text as a GA feature + area: Mapping + type: feature + issues: [] + highlight: + notable: true + title: Release semantic_text as a GA feature + body: semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. + pr: 124669 + - pr: 124676 + summary: TO_LOWER processes all values + area: ES|QL + type: bug + issues: + - 124002 + - pr: 124690 + summary: Avoid creating known_fields for every check in Alias + area: Indices APIs + type: enhancement + issues: [] + - pr: 124722 + summary: Add panama implementations of byte-bit and float-bit script operations + area: Vector Search + type: enhancement + issues: + - 117096 + - pr: 124732 + summary: Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. + area: Infra/Core + type: bug + issues: + - 124667 + - pr: 124738 + summary: Upgrade AWS v2 SDK to 2.30.38 + area: Machine Learning + type: upgrade + issues: [] + - pr: 124739 + summary: Improve rolling up metrics + area: Downsampling + type: enhancement + issues: [] + - pr: 124769 + summary: Migrate `model_version` to `model_id` when parsing persistent elser inference endpoints + area: Machine Learning + type: bug + issues: + - 124675 + - pr: 124782 + summary: "Aggs: Let terms run in global ords mode no match" + area: Aggregations + type: bug + issues: [] + - pr: 124784 + summary: Merge template mappings properly during validation + area: Mapping + type: bug + issues: + - 123372 + - pr: 124823 + summary: Report failures on partial results + area: ES|QL + type: enhancement + issues: [] + - pr: 124841 + summary: Pass `IndexReshardingMetadata` over the wire + area: Distributed + type: bug + issues: [] + - pr: 124843 + summary: Ignore _JAVA_OPTIONS + area: Infra/CLI + type: enhancement + issues: [] + - pr: 124865 + summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." + area: Infra/Core + type: upgrade + issues: [] + - pr: 124873 + summary: Restore V8 REST compatibility around highlight `force_source` parameter + area: Highlighting + type: bug + issues: [] + - pr: 124884 + summary: System data streams are not being upgraded in the feature migration API + area: Infra/Core + type: bug + issues: + - 122949 + - pr: 124898 + summary: Add cache support in `TransportGetAllocationStatsAction` + area: Allocation + type: enhancement + issues: + - 110716 + - pr: 124901 + summary: Calculate concurrent node limit + area: ES|QL + type: feature + issues: [] + - pr: 124913 + summary: Report `original_types` + area: ES|QL + type: enhancement + issues: [] + - pr: 124918 + summary: Fix EQL double invoking listener + area: EQL + type: bug + issues: [] + - pr: 124927 + summary: Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` + area: Mapping + type: enhancement + issues: [] + - pr: 124929 + summary: Include failures in partial response + area: ES|QL + type: enhancement + issues: [] + - pr: 124931 + summary: This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. + area: Snapshot/Restore + type: bug + issues: [] + - pr: 124936 + summary: Indicate when errors represent timeouts + area: Infra/REST API + type: enhancement + issues: [] + - pr: 125002 + summary: Don't generate stacktrace in `TaskCancelledException` + area: Search + type: bug + issues: [] + - pr: 125023 + summary: Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` + area: Machine Learning + type: bug + issues: [] + - pr: 125054 + summary: Truncate `step_info` and error reason in ILM execution state and history + area: ILM+SLM + type: enhancement + issues: + - 124181 + - pr: 125103 + summary: Fix LTR query feature with phrases (and two-phase) queries + area: Ranking + type: bug + issues: [] + - pr: 125117 + summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." + area: Infra/Core + type: upgrade + issues: [] + - pr: 125159 + summary: Update bundled JDK to Java 24 + area: Packaging + type: upgrade + issues: [] + - pr: 125171 + summary: Reindex data stream indices on different nodes + area: Data streams + type: enhancement + issues: [] + - pr: 125204 + summary: Return a Conflict status code if the model deployment is stopped by a user + area: Machine Learning + type: bug + issues: + - 123745 + - pr: 125213 + summary: Run `TransportGetDataStreamOptionsAction` on local node + area: Data streams + type: enhancement + issues: [] + - pr: 125214 + summary: Run `TransportGetDataStreamLifecycleAction` on local node + area: Data streams + type: enhancement + issues: [] + - pr: 125239 + summary: Adding common rerank options to Perform Inference API + area: Machine Learning + type: enhancement + issues: + - 111273 + - pr: 125244 + summary: Disable logging in `ClusterFormationFailureHelper` on shutdown + area: Cluster Coordination + type: bug + issues: + - 105559 + - pr: 125259 + summary: Leverage scorer supplier in `QueryFeatureExtractor` + area: Ranking + type: enhancement + issues: [] + - pr: 125345 + summary: ESQL - date nanos range bug? + area: ES|QL + type: bug + issues: + - 125439 + - pr: 125352 + summary: Fix NPE in rolling over unknown target and return 404 + area: Indices APIs + type: bug + issues: [] + - pr: 125357 + summary: Keep ordinals in conversion functions + area: ES|QL + type: enhancement + issues: [] + - pr: 125370 + summary: Set default similarity for Cohere model to cosine + area: Machine Learning + type: bug + issues: + - 122878 + - pr: 125404 + summary: Check if the anomaly results index has been rolled over + area: Machine Learning + type: upgrade + issues: [] + - pr: 125446 + summary: Fix Semantic Text 8.x Upgrade Bug + area: Mapping + type: bug + issues: [] + - pr: 125529 + summary: Store arrays offsets for boolean fields natively with synthetic source + area: Mapping + type: enhancement + issues: [] + - pr: 125595 + summary: Esql - Fix lucene push down behavior when a range contains nanos and millis + area: ES|QL + type: bug + issues: [] + - pr: 125599 + summary: Allow zero for `rescore_vector.oversample` to indicate by-passing oversample and rescoring + area: Vector Search + type: enhancement + issues: [] + - pr: 125606 + summary: Rename deprecation index template + area: Infra/Logging + type: breaking + issues: + - 125445 + breaking: + area: Logging + title: Rename deprecation index template + details: "The deprecation datastream contains log entries for deprecations that occured while Elasticsearch is running. The deprecation log entries had a mismatch in their dataset name. In order to avoid changing the dataset name in the existing datastream, a new datastream now exists." + impact: "If querying for deprecations previously using the `.logs-deprecation.elasticsearch-default` datastream, you should now use the `.logs-elasticsearch.deprecation-default` datastream." + notable: false + essSettingChange: false + - pr: 125636 + summary: Make `numberOfChannels` consistent with layout map by removing duplicated `ChannelSet` + area: ES|QL + type: bug + issues: [] + - pr: 125650 + summary: Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` + area: Search + type: bug + issues: + - 125483 + - pr: 125659 + summary: Non existing synonyms sets do not fail shard recovery for indices + area: Analysis + type: bug + issues: + - 125603 + - pr: 125666 + summary: Minor-Fixes Support 7x segments as archive in 8x / 9x + area: Search + type: bug + issues: [] + - pr: 125709 + summary: Store arrays offsets for unsigned long fields natively with synthetic source + area: Mapping + type: enhancement + issues: [] + - pr: 125716 + summary: Return appropriate error on null dims update instead of npe + area: Vector Search + type: bug + issues: [] diff --git a/docs/release-notes/changelog-bundles/9.1.0.yml b/docs/release-notes/changelog-bundles/9.1.0.yml new file mode 100644 index 0000000000000..1a6e3e52abdf3 --- /dev/null +++ b/docs/release-notes/changelog-bundles/9.1.0.yml @@ -0,0 +1,1097 @@ +version: 9.1.0 +generated: 2025-03-31T18:44:39.319035Z +changelogs: + - pr: 113757 + summary: Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. + area: Mapping + type: enhancement + issues: [] + - pr: 117642 + summary: Adding endpoint creation validation to `ElasticInferenceService` + area: Machine Learning + type: enhancement + issues: [] + - pr: 119503 + summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. + area: Search + type: bug + issues: + - 117042 + - pr: 119546 + summary: Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields + area: Mapping + type: enhancement + issues: [] + - pr: 119886 + summary: Initial support for unmapped fields + area: ES|QL + type: feature + issues: [] + - pr: 119995 + summary: "apm-data: Use representative count as event.success_count if available" + area: Ingest Node + type: bug + issues: [] + - pr: 120751 + summary: Adding support for binary embedding type to Cohere service embedding type + area: Machine Learning + type: enhancement + issues: [] + - pr: 120774 + summary: Retry ES|QL node requests on shard level failures + area: ES|QL + type: enhancement + issues: [] + - pr: 120869 + summary: Threadpool merge scheduler + area: Engine + type: feature + issues: [] + - pr: 120957 + summary: Introduce `AllocationBalancingRoundSummaryService` + area: Allocation + type: enhancement + issues: [] + - pr: 120998 + summary: ES|QL `change_point` processing command + area: Machine Learning + type: feature + issues: [] + - pr: 121106 + summary: Add `ModelRegistryMetadata` to Cluster State + area: Machine Learning + type: enhancement + issues: [] + - pr: 121240 + summary: Implement runtime skip_unavailable=true + area: ES|QL + type: enhancement + issues: [] + - pr: 121256 + summary: Run `TransportEnrichStatsAction` on local node + area: Ingest Node + type: enhancement + issues: [] + - pr: 121260 + summary: Introduce a pre-mapping logical plan processing step + area: ES|QL + type: enhancement + issues: [] + - pr: 121324 + summary: Support duplicate suggestions in completion field + area: Suggesters + type: bug + issues: + - 82432 + - pr: 121327 + summary: Reduce Data Loss in System Indices Migration + area: Infra/Core + type: bug + issues: [] + - pr: 121370 + summary: Improve SLM Health Indicator to cover missing snapshot + area: ILM+SLM + type: enhancement + issues: [] + - pr: 121392 + summary: Include data streams when converting an existing resource to a system resource + area: Infra/Core + type: bug + issues: [] + - pr: 121548 + summary: Adding support for specifying embedding type to Jina AI service settings + area: Machine Learning + type: enhancement + issues: [] + - pr: 121667 + summary: Add deprecation warning for flush API + area: Machine Learning + type: deprecation + issues: + - 121506 + deprecation: + area: REST API + title: Add deprecation warning for flush API + details: "The anomaly detection job flush API is deprecated since it is only required for the post data API, which was deprecated since 7.11.0." + impact: "This should have a minimal impact on users as the flush API is only required for the post data API, which was deprecated since 7.11.0." + notable: false + essSettingChange: false + - pr: 121784 + summary: Optionally allow text similarity reranking to fail + area: Search + type: enhancement + issues: [] + - pr: 121787 + summary: Added optional parameters to QSTR ES|QL function + area: Search + type: feature + issues: + - 120933 + - pr: 121793 + summary: ES|QL - Add scoring for full text functions disjunctions + area: ES|QL + type: enhancement + issues: [] + - pr: 121805 + summary: Support subset of metrics in aggregate metric double + area: ES|QL + type: enhancement + issues: [] + - pr: 121827 + summary: Updates to allow using Cohere binary embedding response in semantic search queries + area: Machine Learning + type: bug + issues: [] + - pr: 121920 + summary: Account for the `SearchHit` source in circuit breaker + area: Search + type: enhancement + issues: + - 89656 + - pr: 121942 + summary: Allow partial results in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 121948 + summary: Add initial grammar and changes for FORK + area: ES|QL + type: feature + issues: [] + - pr: 121971 + summary: Do not fetch reserved roles from native store when Get Role API is called + area: Authorization + type: enhancement + issues: [] + - pr: 122062 + summary: Upgrade `discovery-ec2` to AWS SDK v2 + area: Discovery-Plugins + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Upgrade `discovery-ec2` to AWS SDK v2 + details: |2- + + In earlier versions of {es} the `discovery-ec2` plugin was based on the AWS SDK v1. AWS will withdraw support for this SDK before the end of the life of {es} {minor-version} so we must migrate to the newer AWS SDK v2. + Unfortunately there are several differences between the two AWS SDK versions which may require you to adjust your system configuration when upgrading to {es} {minor-version} or later. These differences include, but may not be limited to, the following items. + * AWS SDK v2 does not support the EC2 IMDSv1 protocol. + * AWS SDK v2 does not support the `aws.secretKey` or + `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties. + + * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so + the `discovery.ec2.protocol` setting is no longer effective. + + * AWS SDK v2 does not accept an access key without a secret key or vice + versa. + impact: |2- + + If you use the `discovery-ec2` plugin, test your upgrade thoroughly before upgrading any production workloads. + Adapt your configuration to the new SDK functionality. This includes, but may not be limited to, the following items. + * If you use IMDS to determine the availability zone of a node or to obtain + credentials for accessing the EC2 API, ensure that it supports the IMDSv2 + protocol. + + * If applicable, discontinue use of the `aws.secretKey` and + `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties. + + * If applicable, specify that you wish to use the insecure HTTP protocol to + access the EC2 API by setting `discovery.ec2.endpoint` to a URL which + starts with `http://`. + + * Either supply both an access key and a secret key using the keystore + settings `discovery.ec2.access_key` and `discovery.ec2.secret_key`, or + configure neither of these settings. + notable: true + essSettingChange: false + - pr: 122066 + summary: Adding elser default endpoint for EIS + area: Machine Learning + type: enhancement + issues: [] + - pr: 122074 + summary: "If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias" + area: Transform + type: bug + issues: + - 121913 + - pr: 122134 + summary: Adding integration for VoyageAI embeddings and rerank models + area: Machine Learning + type: enhancement + issues: [] + - pr: 122218 + summary: Integrate with `DeepSeek` API + area: Machine Learning + type: enhancement + issues: [] + - pr: 122224 + summary: Enable the use of nested field type with index.mode=time_series + area: Mapping + type: enhancement + issues: + - 120874 + - pr: 122250 + summary: "ESQL: Align `RENAME` behavior with `EVAL` for sequential processing" + area: ES|QL + type: enhancement + issues: + - 121739 + - pr: 122257 + summary: Revive inlinestats + area: ES|QL + type: bug + issues: [] + - pr: 122272 + summary: "[Inference API] Rename `model_id` prop to model in EIS sparse inference request body" + area: Inference + type: enhancement + issues: [] + - pr: 122278 + summary: Fix serialising the inference update request + area: Machine Learning + type: bug + issues: [] + - pr: 122280 + summary: Use `FallbackSyntheticSourceBlockLoader` for number fields + area: Mapping + type: enhancement + issues: [] + - pr: 122293 + summary: Add enterprise license check to inference action for semantic text fields + area: Machine Learning + type: bug + issues: [] + - pr: 122326 + summary: System Index Migration Failure Results in a Non-Recoverable State + area: Infra/Core + type: bug + issues: [] + - pr: 122357 + summary: Handle search timeout in `SuggestPhase` + area: Search + type: bug + issues: + - 122186 + - pr: 122365 + summary: Fix handling of auto expand replicas for stateless indices + area: Search + type: bug + issues: [] + - pr: 122381 + summary: Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields + area: Vector Search + type: enhancement + issues: + - 117274 + - pr: 122390 + summary: Add health indicator impact to `HealthPeriodicLogger` + area: Health + type: enhancement + issues: [] + - pr: 122409 + summary: Allow setting the `type` in the reroute processor + area: Ingest Node + type: enhancement + issues: + - 121553 + - pr: 122427 + summary: Improve size limiting string message + area: Infra/Core + type: enhancement + issues: [] + - pr: 122431 + summary: Upgrade AWS SDK to v1.12.746 + area: Snapshot/Restore + type: upgrade + issues: [] + - pr: 122458 + summary: '`DesiredBalanceReconciler` always returns `AllocationStats`' + area: Allocation + type: bug + issues: [] + - pr: 122459 + summary: Double parameter markers for identifiers + area: ES|QL + type: enhancement + issues: [] + - pr: 122486 + summary: Add index mode to get data stream API + area: Data streams + type: enhancement + issues: [] + - pr: 122575 + summary: Return an empty suggestion when suggest phase times out + area: Suggesters + type: bug + issues: + - 122548 + - pr: 122586 + summary: "ESQL: Fix inconsistent results in using scaled_float field" + area: ES|QL + type: bug + issues: + - 122547 + - pr: 122601 + summary: Implicit numeric casting for CASE/GREATEST/LEAST + area: ES|QL + type: bug + issues: + - 121890 + - pr: 122606 + summary: Avoid serializing empty `_source` fields in mappings + area: Mapping + type: bug + issues: [] + - pr: 122615 + summary: Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. + area: Mapping + type: enhancement + issues: + - 116726 + - pr: 122637 + summary: Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields + area: Mapping + type: enhancement + issues: [] + - pr: 122638 + summary: Expose `input_type` option at root level for `text_embedding` task type in Perform Inference API + area: Machine Learning + type: enhancement + issues: + - 117856 + - pr: 122653 + summary: Knn vector rescoring to sort score docs + area: Vector Search + type: bug + issues: + - 119711 + - pr: 122660 + summary: Render `aggregate_metric_double` + area: ES|QL + type: enhancement + issues: [] + - pr: 122708 + summary: Support partial results in CCS in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 122731 + summary: Fork post-snapshot-delete cleanup off master thread + area: Snapshot/Restore + type: bug + issues: [] + - pr: 122762 + summary: "ESQL: Remove estimated row size assertion" + area: ES|QL + type: bug + issues: + - 121535 + - pr: 122821 + summary: Fix functions emitting warnings with no source + area: ES|QL + type: bug + issues: + - 122588 + - pr: 122823 + summary: Prevent Query Rule Creation with Invalid Numeric Match Criteria + area: Relevance + type: bug + issues: [] + - pr: 122852 + summary: Run `TransportGetDataStreamsAction` on local node + area: Data streams + type: enhancement + issues: [] + - pr: 122857 + summary: Run `TransportGetWatcherSettingsAction` on local node + area: Watcher + type: enhancement + issues: [] + - pr: 122860 + summary: Improved error message when index field type is invalid + area: Mapping + type: enhancement + issues: [] + - pr: 122885 + summary: Run `TransportExplainLifecycleAction` on local node + area: ILM+SLM + type: enhancement + issues: [] + - pr: 122886 + summary: Add support to VALUES aggregation for spatial types + area: ES|QL + type: bug + issues: + - 122413 + - pr: 122890 + summary: Introduce `allow_partial_results` setting in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 122891 + summary: Pragma to load from stored fields + area: ES|QL + type: enhancement + issues: [] + - pr: 122905 + summary: Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover + area: Data streams + type: bug + issues: [] + - pr: 122921 + summary: Run `TransportGetMappingsAction` on local node + area: Indices APIs + type: enhancement + issues: [] + - pr: 122933 + summary: Run XPack usage actions on local node + area: Stats + type: enhancement + issues: [] + - pr: 122938 + summary: Fix geoip databases index access after system feature migration (again) + area: Ingest Node + type: bug + issues: [] + - pr: 122951 + summary: Updates the deprecation info API to not warn about system indices and data streams + area: Indices APIs + type: bug + issues: [] + - pr: 122960 + summary: Deprecate Behavioral Analytics CRUD apis + area: Search + type: deprecation + issues: [] + deprecation: + area: Search + title: Deprecate Behavioral Analytics CRUD apis + details: "Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. The APIs will still work for now, but will emit warning headers that the API has been deprecated." + impact: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. + notable: false + essSettingChange: false + - pr: 122991 + summary: "GCS blob store: add `OperationPurpose/Operation` stats counters" + area: Snapshot/Restore + type: enhancement + issues: [] + - pr: 122999 + summary: Store arrays offsets for ip fields natively with synthetic source + area: Mapping + type: enhancement + issues: [] + - pr: 123010 + summary: Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` + area: Engine + type: bug + issues: + - 122974 + - pr: 123076 + summary: Retry on streaming errors + area: Machine Learning + type: bug + issues: [] + - pr: 123085 + summary: Remove duplicated nested commands + area: ES|QL + type: bug + issues: [] + - pr: 123150 + summary: Limit the number of chunks for semantic text to prevent high memory usage + area: Machine Learning + type: feature + issues: [] + - pr: 123155 + summary: Add `ElasticInferenceServiceCompletionServiceSettings` + area: Machine Learning + type: bug + issues: [] + - pr: 123187 + summary: Add bit vector support to semantic text + area: Vector Search + type: enhancement + issues: [] + - pr: 123197 + summary: Fix early termination in `LuceneSourceOperator` + area: ES|QL + type: bug + issues: [] + - pr: 123272 + summary: Set Connect Timeout to 5s + area: Machine Learning + type: bug + issues: [] + - pr: 123290 + summary: Fix Driver status iterations and `cpuTime` + area: ES|QL + type: enhancement + issues: + - 122967 + - pr: 123296 + summary: Avoid over collecting in Limit or Lucene Operator + area: ES|QL + type: bug + issues: [] + - pr: 123381 + summary: Push down `StartsWith` and `EndsWith` functions to Lucene + area: ES|QL + type: enhancement + issues: + - 123067 + - pr: 123396 + summary: Add initial grammar and planning for RRF (snapshot) + area: ES|QL + type: feature + issues: [] + - pr: 123427 + summary: Reduce iteration complexity for plan traversal + area: ES|QL + type: bug + issues: [] + - pr: 123460 + summary: "ES|QL: Support `::date` in inline cast" + area: ES|QL + type: enhancement + issues: + - 116746 + - pr: 123492 + summary: Fix function registry concurrency issues on constructor + area: ES|QL + type: bug + issues: + - 123430 + - pr: 123543 + summary: Enhance memory accounting for document expansion and introduce max document size limit + area: CRUD + type: enhancement + issues: [] + - pr: 123569 + summary: Abort pending deletion on `IndicesService` close + area: Store + type: enhancement + issues: [] + - pr: 123588 + summary: Give Kibana user 'all' permissions for .entity_analytics.* indices + area: Infra/Core + type: enhancement + issues: [] + - pr: 123589 + summary: Revive some more of inlinestats functionality + area: ES|QL + type: bug + issues: [] + - pr: 123600 + summary: Drop `TLS_RSA` cipher support for JDK 24 + area: TLS + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Drop `TLS_RSA` cipher support for JDK 24 + details: "This change removes `TLS_RSA` ciphers from the list of default supported ciphers, for Elasticsearch deployments running on JDK 24." + impact: "The dropped ciphers are `TLS_RSA_WITH_AES_256_GCM_SHA384`, `TLS_RSA_WITH_AES_128_GCM_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA`, and `TLS_RSA_WITH_AES_128_CBC_SHA`. TLS connections to Elasticsearch using these ciphers will no longer work. Please configure your clients to use one of supported cipher suites." + notable: false + essSettingChange: false + - pr: 123630 + summary: Limit number of suppressed S3 deletion errors + area: Snapshot/Restore + type: bug + issues: + - 123354 + - pr: 123712 + summary: Process ILM cluster state updates on another thread + area: ILM+SLM + type: enhancement + issues: [] + - pr: 123728 + summary: Allow skip shards with `_tier` and `_index` in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 123743 + summary: Adjust exception thrown when unable to load hunspell dict + area: Analysis + type: bug + issues: [] + - pr: 123757 + summary: Fix concurrency issue in `ScriptSortBuilder` + area: Search + type: bug + issues: [] + - pr: 123761 + summary: Have create index return a bad request on poor formatting + area: Infra/Core + type: bug + issues: [] + - pr: 123763 + summary: Skip semantic_text embedding generation when no content is provided. + area: Relevance + type: enhancement + issues: [] + - pr: 123852 + summary: Retry when the server can't be resolved (Google Cloud Storage) + area: Snapshot/Restore + type: enhancement + issues: [] + - pr: 123890 + summary: Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions + area: Search + type: bug + issues: [] + - pr: 124001 + summary: Use a must boolean statement when pushing down to Lucene when scoring is also needed + area: ES|QL + type: bug + issues: + - 123967 + - pr: 124025 + summary: "[Inference API] Propagate product use case http header to EIS" + area: Machine Learning + type: enhancement + issues: [] + - pr: 124048 + summary: Handle long overflow in dates + area: Search + type: bug + issues: + - 112483 + - pr: 124050 + summary: Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields + area: Mapping + type: enhancement + issues: [] + - pr: 124062 + summary: Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0 + area: Snapshot/Restore + type: upgrade + issues: [] + - pr: 124094 + summary: ES|QL slow log + area: ES|QL + type: enhancement + issues: [] + - pr: 124107 + summary: Avoid hoarding cluster state references during rollover + area: Indices APIs + type: bug + issues: + - 123893 + - pr: 124149 + summary: Retry ILM async action after reindexing data stream + area: Data streams + type: enhancement + issues: [] + - pr: 124177 + summary: "Improve error message for ( and [" + area: ES|QL + type: bug + issues: + - 124145 + - pr: 124225 + summary: "[Inference API] Fix output stream ordering in `InferenceActionProxy`" + area: Machine Learning + type: bug + issues: [] + - pr: 124313 + summary: Optimize memory usage in `ShardBulkInferenceActionFilter` + area: Search + type: enhancement + issues: [] + - pr: 124335 + summary: Change the order of the optimization rules + area: ES|QL + type: bug + issues: [] + - pr: 124363 + summary: Set cause on create index request in create from action + area: Data streams + type: enhancement + issues: [] + - pr: 124394 + summary: Avoid `NamedWritable` in block serialization + area: ES|QL + type: enhancement + issues: [] + - pr: 124424 + summary: Lazy collection copying during node transform + area: ES|QL + type: bug + issues: [] + - pr: 124446 + summary: "ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate`" + area: ES|QL + type: bug + issues: + - 124311 + - pr: 124451 + summary: Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. + area: Downsampling + type: bug + issues: [] + - pr: 124477 + summary: Improve downsample performance by buffering docids and do bulk processing + area: Downsampling + type: enhancement + issues: [] + - pr: 124527 + summary: Avoid potentially throwing calls to Task#getDescription in model download + area: Machine Learning + type: bug + issues: [] + - pr: 124540 + summary: "ES|QL: Fix scoring for full text functions" + area: ES|QL + type: bug + issues: [] + - pr: 124544 + summary: Bump nimbus-jose-jwt to 10.0.2 + area: Security + type: upgrade + issues: [] + - pr: 124564 + summary: Restore `TextSimilarityRankBuilder` XContent output + area: Ranking + type: bug + issues: [] + - pr: 124574 + summary: Allow passing several reserved state chunks in single process call + area: Infra/Settings + type: enhancement + issues: [] + - pr: 124581 + summary: New `vector_rescore` parameter as a quantized index type option + area: Vector Search + type: enhancement + issues: [] + - pr: 124594 + summary: Store arrays offsets for numeric fields natively with synthetic source + area: Mapping + type: enhancement + issues: [] + - pr: 124595 + summary: '`ToAggregateMetricDouble` function' + area: ES|QL + type: enhancement + issues: [] + - pr: 124604 + summary: Fix geoip databases index access after system feature migration (take 3) + area: Ingest Node + type: bug + issues: [] + - pr: 124610 + summary: Remove page alignment in exchange sink + area: ES|QL + type: enhancement + issues: [] + - pr: 124611 + summary: Reuse child `outputSet` inside the plan where possible + area: ES|QL + type: enhancement + issues: [] + - pr: 124638 + summary: Provide model size statistics as soon as an anomaly detection job is opened + area: Machine Learning + type: bug + issues: + - 121168 + - pr: 124651 + summary: Fix system data streams to be restorable from a snapshot + area: Infra/Core + type: bug + issues: + - 89261 + - pr: 124662 + summary: Let MLTQuery throw IAE when no analyzer is set + area: Search + type: bug + issues: + - 124562 + - pr: 124669 + summary: Release semantic_text as a GA feature + area: Mapping + type: feature + issues: [] + highlight: + notable: true + title: Release semantic_text as a GA feature + body: semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. + pr: 124669 + - pr: 124676 + summary: TO_LOWER processes all values + area: ES|QL + type: bug + issues: + - 124002 + - pr: 124690 + summary: Avoid creating known_fields for every check in Alias + area: Indices APIs + type: enhancement + issues: [] + - pr: 124722 + summary: Add panama implementations of byte-bit and float-bit script operations + area: Vector Search + type: enhancement + issues: + - 117096 + - pr: 124732 + summary: Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. + area: Infra/Core + type: bug + issues: + - 124667 + - pr: 124738 + summary: Upgrade AWS v2 SDK to 2.30.38 + area: Machine Learning + type: upgrade + issues: [] + - pr: 124739 + summary: Improve rolling up metrics + area: Downsampling + type: enhancement + issues: [] + - pr: 124769 + summary: Migrate `model_version` to `model_id` when parsing persistent elser inference endpoints + area: Machine Learning + type: bug + issues: + - 124675 + - pr: 124782 + summary: "Aggs: Let terms run in global ords mode no match" + area: Aggregations + type: bug + issues: [] + - pr: 124784 + summary: Merge template mappings properly during validation + area: Mapping + type: bug + issues: + - 123372 + - pr: 124823 + summary: Report failures on partial results + area: ES|QL + type: enhancement + issues: [] + - pr: 124841 + summary: Pass `IndexReshardingMetadata` over the wire + area: Distributed + type: bug + issues: [] + - pr: 124843 + summary: Ignore _JAVA_OPTIONS + area: Infra/CLI + type: enhancement + issues: [] + - pr: 124873 + summary: Restore V8 REST compatibility around highlight `force_source` parameter + area: Highlighting + type: bug + issues: [] + - pr: 124898 + summary: Add cache support in `TransportGetAllocationStatsAction` + area: Allocation + type: enhancement + issues: + - 110716 + - pr: 124901 + summary: Calculate concurrent node limit + area: ES|QL + type: feature + issues: [] + - pr: 124913 + summary: Report `original_types` + area: ES|QL + type: enhancement + issues: [] + - pr: 124918 + summary: Fix EQL double invoking listener + area: EQL + type: bug + issues: [] + - pr: 124927 + summary: Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` + area: Mapping + type: enhancement + issues: [] + - pr: 124929 + summary: Include failures in partial response + area: ES|QL + type: enhancement + issues: [] + - pr: 124931 + summary: This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. + area: Snapshot/Restore + type: bug + issues: [] + - pr: 124936 + summary: Indicate when errors represent timeouts + area: Infra/REST API + type: enhancement + issues: [] + - pr: 124958 + summary: Catch parsing exception + area: ES|QL + type: bug + issues: + - 119025 + - pr: 125002 + summary: Don't generate stacktrace in `TaskCancelledException` + area: Search + type: bug + issues: [] + - pr: 125023 + summary: Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` + area: Machine Learning + type: bug + issues: [] + - pr: 125054 + summary: Truncate `step_info` and error reason in ILM execution state and history + area: ILM+SLM + type: enhancement + issues: + - 124181 + - pr: 125103 + summary: Fix LTR query feature with phrases (and two-phase) queries + area: Ranking + type: bug + issues: [] + - pr: 125117 + summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." + area: Infra/Core + type: upgrade + issues: [] + - pr: 125159 + summary: Update bundled JDK to Java 24 + area: Packaging + type: upgrade + issues: [] + - pr: 125171 + summary: Reindex data stream indices on different nodes + area: Data streams + type: enhancement + issues: [] + - pr: 125204 + summary: Return a Conflict status code if the model deployment is stopped by a user + area: Machine Learning + type: bug + issues: + - 123745 + - pr: 125213 + summary: Run `TransportGetDataStreamOptionsAction` on local node + area: Data streams + type: enhancement + issues: [] + - pr: 125214 + summary: Run `TransportGetDataStreamLifecycleAction` on local node + area: Data streams + type: enhancement + issues: [] + - pr: 125239 + summary: Adding common rerank options to Perform Inference API + area: Machine Learning + type: enhancement + issues: + - 111273 + - pr: 125244 + summary: Disable logging in `ClusterFormationFailureHelper` on shutdown + area: Cluster Coordination + type: bug + issues: + - 105559 + - pr: 125259 + summary: Leverage scorer supplier in `QueryFeatureExtractor` + area: Ranking + type: enhancement + issues: [] + - pr: 125345 + summary: ESQL - date nanos range bug? + area: ES|QL + type: bug + issues: + - 125439 + - pr: 125352 + summary: Fix NPE in rolling over unknown target and return 404 + area: Indices APIs + type: bug + issues: [] + - pr: 125357 + summary: Keep ordinals in conversion functions + area: ES|QL + type: enhancement + issues: [] + - pr: 125370 + summary: Set default similarity for Cohere model to cosine + area: Machine Learning + type: bug + issues: + - 122878 + - pr: 125404 + summary: Check if the anomaly results index has been rolled over + area: Machine Learning + type: upgrade + issues: [] + - pr: 125446 + summary: Fix Semantic Text 8.x Upgrade Bug + area: Mapping + type: bug + issues: [] + - pr: 125529 + summary: Store arrays offsets for boolean fields natively with synthetic source + area: Mapping + type: enhancement + issues: [] + - pr: 125595 + summary: Esql - Fix lucene push down behavior when a range contains nanos and millis + area: ES|QL + type: bug + issues: [] + - pr: 125599 + summary: Allow zero for `rescore_vector.oversample` to indicate by-passing oversample and rescoring + area: Vector Search + type: enhancement + issues: [] + - pr: 125606 + summary: Rename deprecation index template + area: Infra/Logging + type: breaking + issues: + - 125445 + breaking: + area: Logging + title: Rename deprecation index template + details: "The deprecation datastream contains log entries for deprecations that occured while Elasticsearch is running. The deprecation log entries had a mismatch in their dataset name. In order to avoid changing the dataset name in the existing datastream, a new datastream now exists." + impact: "If querying for deprecations previously using the `.logs-deprecation.elasticsearch-default` datastream, you should now use the `.logs-elasticsearch.deprecation-default` datastream." + notable: false + essSettingChange: false + - pr: 125636 + summary: Make `numberOfChannels` consistent with layout map by removing duplicated `ChannelSet` + area: ES|QL + type: bug + issues: [] + - pr: 125650 + summary: Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` + area: Search + type: bug + issues: + - 125483 + - pr: 125659 + summary: Non existing synonyms sets do not fail shard recovery for indices + area: Analysis + type: bug + issues: + - 125603 + - pr: 125666 + summary: Minor-Fixes Support 7x segments as archive in 8x / 9x + area: Search + type: bug + issues: [] + - pr: 125709 + summary: Store arrays offsets for unsigned long fields natively with synthetic source + area: Mapping + type: enhancement + issues: [] + - pr: 125716 + summary: Return appropriate error on null dims update instead of npe + area: Vector Search + type: bug + issues: [] diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 2f26a1a0f141c..f819f2df12c8f 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -3,8 +3,6 @@ navigation_title: "Elasticsearch" mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-9.1.0.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-9.1.html --- # Elasticsearch release notes [elasticsearch-release-notes] @@ -24,9 +22,16 @@ To check for security updates, go to [Security announcements for the Elastic sta % * ## 9.1.0 [elasticsearch-910-release-notes] -**Release date:** April 01, 2025 -### Highlights [elasticsearch-910-highlights] +### Fixes [elasticsearch-910-fixes] + +ES|QL: +* Catch parsing exception [#124958](https://github.com/elastic/elasticsearch/pull/124958) (issue: [#124958](https://github.com/elastic/elasticsearch/pull/124958)) + + +## 9.0.0 [elasticsearch-900-release-notes] + +### Highlights [elasticsearch-900-highlights] ::::{dropdown} Release semantic_text as a GA feature semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. @@ -34,43 +39,58 @@ semantic_text is now an official GA (generally available) feature! This field ty For more information, check [PR #124669](https://github.com/elastic/elasticsearch/pull/124669). :::: -### Features and enhancements [elasticsearch-910-features-enhancements] +### Features and enhancements [elasticsearch-900-features-enhancements] Allocation: * Add cache support in `TransportGetAllocationStatsAction` [#124898](https://github.com/elastic/elasticsearch/pull/124898) (issue: [#124898](https://github.com/elastic/elasticsearch/pull/124898)) * Introduce `AllocationBalancingRoundSummaryService` [#120957](https://github.com/elastic/elasticsearch/pull/120957) +Authorization: +* Do not fetch reserved roles from native store when Get Role API is called [#121971](https://github.com/elastic/elasticsearch/pull/121971) + CRUD: * Enhance memory accounting for document expansion and introduce max document size limit [#123543](https://github.com/elastic/elasticsearch/pull/123543) Data streams: * Add index mode to get data stream API [#122486](https://github.com/elastic/elasticsearch/pull/122486) * Reindex data stream indices on different nodes [#125171](https://github.com/elastic/elasticsearch/pull/125171) +* Retry ILM async action after reindexing data stream [#124149](https://github.com/elastic/elasticsearch/pull/124149) * Run `TransportGetDataStreamLifecycleAction` on local node [#125214](https://github.com/elastic/elasticsearch/pull/125214) * Run `TransportGetDataStreamOptionsAction` on local node [#125213](https://github.com/elastic/elasticsearch/pull/125213) * Run `TransportGetDataStreamsAction` on local node [#122852](https://github.com/elastic/elasticsearch/pull/122852) +* Set cause on create index request in create from action [#124363](https://github.com/elastic/elasticsearch/pull/124363) + +Downsampling: +* Improve downsample performance by buffering docids and do bulk processing [#124477](https://github.com/elastic/elasticsearch/pull/124477) +* Improve rolling up metrics [#124739](https://github.com/elastic/elasticsearch/pull/124739) ES|QL: * Add initial grammar and changes for FORK [#121948](https://github.com/elastic/elasticsearch/pull/121948) * Add initial grammar and planning for RRF (snapshot) [#123396](https://github.com/elastic/elasticsearch/pull/123396) * Allow partial results in ES|QL [#121942](https://github.com/elastic/elasticsearch/pull/121942) +* Allow skip shards with `_tier` and `_index` in ES|QL [#123728](https://github.com/elastic/elasticsearch/pull/123728) * Avoid `NamedWritable` in block serialization [#124394](https://github.com/elastic/elasticsearch/pull/124394) * Calculate concurrent node limit [#124901](https://github.com/elastic/elasticsearch/pull/124901) * Double parameter markers for identifiers [#122459](https://github.com/elastic/elasticsearch/pull/122459) * ESQL: Align `RENAME` behavior with `EVAL` for sequential processing [#122250](https://github.com/elastic/elasticsearch/pull/122250) (issue: [#122250](https://github.com/elastic/elasticsearch/pull/122250)) +* ES|QL - Add scoring for full text functions disjunctions [#121793](https://github.com/elastic/elasticsearch/pull/121793) * ES|QL slow log [#124094](https://github.com/elastic/elasticsearch/pull/124094) * ES|QL: Support `::date` in inline cast [#123460](https://github.com/elastic/elasticsearch/pull/123460) (issue: [#123460](https://github.com/elastic/elasticsearch/pull/123460)) +* Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: [#123290](https://github.com/elastic/elasticsearch/pull/123290)) * Implement runtime skip_unavailable=true [#121240](https://github.com/elastic/elasticsearch/pull/121240) * Include failures in partial response [#124929](https://github.com/elastic/elasticsearch/pull/124929) +* Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) * Introduce `allow_partial_results` setting in ES|QL [#122890](https://github.com/elastic/elasticsearch/pull/122890) * Introduce a pre-mapping logical plan processing step [#121260](https://github.com/elastic/elasticsearch/pull/121260) * Keep ordinals in conversion functions [#125357](https://github.com/elastic/elasticsearch/pull/125357) * Pragma to load from stored fields [#122891](https://github.com/elastic/elasticsearch/pull/122891) +* Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: [#123381](https://github.com/elastic/elasticsearch/pull/123381)) * Remove page alignment in exchange sink [#124610](https://github.com/elastic/elasticsearch/pull/124610) * Render `aggregate_metric_double` [#122660](https://github.com/elastic/elasticsearch/pull/122660) * Report `original_types` [#124913](https://github.com/elastic/elasticsearch/pull/124913) * Report failures on partial results [#124823](https://github.com/elastic/elasticsearch/pull/124823) * Retry ES|QL node requests on shard level failures [#120774](https://github.com/elastic/elasticsearch/pull/120774) +* Reuse child `outputSet` inside the plan where possible [#124611](https://github.com/elastic/elasticsearch/pull/124611) * Support partial results in CCS in ES|QL [#122708](https://github.com/elastic/elasticsearch/pull/122708) * Support subset of metrics in aggregate metric double [#121805](https://github.com/elastic/elasticsearch/pull/121805) * `ToAggregateMetricDouble` function [#124595](https://github.com/elastic/elasticsearch/pull/124595) @@ -94,14 +114,23 @@ Indices APIs: Inference: * [Inference API] Rename `model_id` prop to model in EIS sparse inference request body [#122272](https://github.com/elastic/elasticsearch/pull/122272) +Infra/CLI: +* Ignore _JAVA_OPTIONS [#124843](https://github.com/elastic/elasticsearch/pull/124843) + Infra/Core: * Give Kibana user 'all' permissions for .entity_analytics.* indices [#123588](https://github.com/elastic/elasticsearch/pull/123588) +* Improve size limiting string message [#122427](https://github.com/elastic/elasticsearch/pull/122427) +* Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#124865](https://github.com/elastic/elasticsearch/pull/124865) * Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#125117](https://github.com/elastic/elasticsearch/pull/125117) +Infra/REST API: +* Indicate when errors represent timeouts [#124936](https://github.com/elastic/elasticsearch/pull/124936) + Infra/Settings: * Allow passing several reserved state chunks in single process call [#124574](https://github.com/elastic/elasticsearch/pull/124574) Ingest Node: +* Allow setting the `type` in the reroute processor [#122409](https://github.com/elastic/elasticsearch/pull/122409) (issue: [#122409](https://github.com/elastic/elasticsearch/pull/122409)) * Run `TransportEnrichStatsAction` on local node [#121256](https://github.com/elastic/elasticsearch/pull/121256) Machine Learning: @@ -136,6 +165,9 @@ Mapping: * Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields [#124050](https://github.com/elastic/elasticsearch/pull/124050) * Use `FallbackSyntheticSourceBlockLoader` for number fields [#122280](https://github.com/elastic/elasticsearch/pull/122280) +Packaging: +* Update bundled JDK to Java 24 [#125159](https://github.com/elastic/elasticsearch/pull/125159) + Ranking: * Leverage scorer supplier in `QueryFeatureExtractor` [#125259](https://github.com/elastic/elasticsearch/pull/125259) @@ -144,12 +176,17 @@ Relevance: Search: * Account for the `SearchHit` source in circuit breaker [#121920](https://github.com/elastic/elasticsearch/pull/121920) (issue: [#121920](https://github.com/elastic/elasticsearch/pull/121920)) +* Added optional parameters to QSTR ES|QL function [#121787](https://github.com/elastic/elasticsearch/pull/121787) (issue: [#121787](https://github.com/elastic/elasticsearch/pull/121787)) * Optimize memory usage in `ShardBulkInferenceActionFilter` [#124313](https://github.com/elastic/elasticsearch/pull/124313) * Optionally allow text similarity reranking to fail [#121784](https://github.com/elastic/elasticsearch/pull/121784) +Security: +* Bump nimbus-jose-jwt to 10.0.2 [#124544](https://github.com/elastic/elasticsearch/pull/124544) + Snapshot/Restore: * GCS blob store: add `OperationPurpose/Operation` stats counters [#122991](https://github.com/elastic/elasticsearch/pull/122991) * Retry when the server can't be resolved (Google Cloud Storage) [#123852](https://github.com/elastic/elasticsearch/pull/123852) +* Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) * Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0 [#124062](https://github.com/elastic/elasticsearch/pull/124062) Stats: @@ -168,7 +205,7 @@ Vector Search: Watcher: * Run `TransportGetWatcherSettingsAction` on local node [#122857](https://github.com/elastic/elasticsearch/pull/122857) -### Fixes [elasticsearch-910-fixes] +### Fixes [elasticsearch-900-fixes] Aggregations: * Aggs: Let terms run in global ords mode no match [#124782](https://github.com/elastic/elasticsearch/pull/124782) @@ -177,67 +214,125 @@ Allocation: * `DesiredBalanceReconciler` always returns `AllocationStats` [#122458](https://github.com/elastic/elasticsearch/pull/122458) Analysis: +* Adjust exception thrown when unable to load hunspell dict [#123743](https://github.com/elastic/elasticsearch/pull/123743) * Non existing synonyms sets do not fail shard recovery for indices [#125659](https://github.com/elastic/elasticsearch/pull/125659) (issue: [#125659](https://github.com/elastic/elasticsearch/pull/125659)) Cluster Coordination: * Disable logging in `ClusterFormationFailureHelper` on shutdown [#125244](https://github.com/elastic/elasticsearch/pull/125244) (issue: [#125244](https://github.com/elastic/elasticsearch/pull/125244)) +Data streams: +* Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover [#122905](https://github.com/elastic/elasticsearch/pull/122905) + Distributed: * Pass `IndexReshardingMetadata` over the wire [#124841](https://github.com/elastic/elasticsearch/pull/124841) +Downsampling: +* Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. [#124451](https://github.com/elastic/elasticsearch/pull/124451) + EQL: * Fix EQL double invoking listener [#124918](https://github.com/elastic/elasticsearch/pull/124918) ES|QL: +* Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: [#122886](https://github.com/elastic/elasticsearch/pull/122886)) +* Avoid over collecting in Limit or Lucene Operator [#123296](https://github.com/elastic/elasticsearch/pull/123296) +* Change the order of the optimization rules [#124335](https://github.com/elastic/elasticsearch/pull/124335) * ESQL - date nanos range bug? [#125345](https://github.com/elastic/elasticsearch/pull/125345) (issue: [#125345](https://github.com/elastic/elasticsearch/pull/125345)) * ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate` [#124446](https://github.com/elastic/elasticsearch/pull/124446) (issue: [#124446](https://github.com/elastic/elasticsearch/pull/124446)) * ESQL: Fix inconsistent results in using scaled_float field [#122586](https://github.com/elastic/elasticsearch/pull/122586) (issue: [#122586](https://github.com/elastic/elasticsearch/pull/122586)) +* ESQL: Remove estimated row size assertion [#122762](https://github.com/elastic/elasticsearch/pull/122762) (issue: [#122762](https://github.com/elastic/elasticsearch/pull/122762)) +* ES|QL: Fix scoring for full text functions [#124540](https://github.com/elastic/elasticsearch/pull/124540) * Esql - Fix lucene push down behavior when a range contains nanos and millis [#125595](https://github.com/elastic/elasticsearch/pull/125595) +* Fix early termination in `LuceneSourceOperator` [#123197](https://github.com/elastic/elasticsearch/pull/123197) +* Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: [#123492](https://github.com/elastic/elasticsearch/pull/123492)) * Fix functions emitting warnings with no source [#122821](https://github.com/elastic/elasticsearch/pull/122821) (issue: [#122821](https://github.com/elastic/elasticsearch/pull/122821)) +* Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: [#122601](https://github.com/elastic/elasticsearch/pull/122601)) * Improve error message for ( and [ [#124177](https://github.com/elastic/elasticsearch/pull/124177) (issue: [#124177](https://github.com/elastic/elasticsearch/pull/124177)) +* Lazy collection copying during node transform [#124424](https://github.com/elastic/elasticsearch/pull/124424) * Make `numberOfChannels` consistent with layout map by removing duplicated `ChannelSet` [#125636](https://github.com/elastic/elasticsearch/pull/125636) +* Reduce iteration complexity for plan traversal [#123427](https://github.com/elastic/elasticsearch/pull/123427) * Remove duplicated nested commands [#123085](https://github.com/elastic/elasticsearch/pull/123085) +* Revive inlinestats [#122257](https://github.com/elastic/elasticsearch/pull/122257) +* Revive some more of inlinestats functionality [#123589](https://github.com/elastic/elasticsearch/pull/123589) * TO_LOWER processes all values [#124676](https://github.com/elastic/elasticsearch/pull/124676) (issue: [#124676](https://github.com/elastic/elasticsearch/pull/124676)) +* Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: [#124001](https://github.com/elastic/elasticsearch/pull/124001)) + +Engine: +* Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: [#123010](https://github.com/elastic/elasticsearch/pull/123010)) + +Highlighting: +* Restore V8 REST compatibility around highlight `force_source` parameter [#124873](https://github.com/elastic/elasticsearch/pull/124873) Indices APIs: +* Avoid hoarding cluster state references during rollover [#124107](https://github.com/elastic/elasticsearch/pull/124107) (issue: [#124107](https://github.com/elastic/elasticsearch/pull/124107)) * Fix NPE in rolling over unknown target and return 404 [#125352](https://github.com/elastic/elasticsearch/pull/125352) +* Updates the deprecation info API to not warn about system indices and data streams [#122951](https://github.com/elastic/elasticsearch/pull/122951) Infra/Core: +* Fix system data streams to be restorable from a snapshot [#124651](https://github.com/elastic/elasticsearch/pull/124651) (issue: [#124651](https://github.com/elastic/elasticsearch/pull/124651)) +* Have create index return a bad request on poor formatting [#123761](https://github.com/elastic/elasticsearch/pull/123761) +* Include data streams when converting an existing resource to a system resource [#121392](https://github.com/elastic/elasticsearch/pull/121392) +* Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. [#124732](https://github.com/elastic/elasticsearch/pull/124732) (issue: [#124732](https://github.com/elastic/elasticsearch/pull/124732)) * Reduce Data Loss in System Indices Migration [#121327](https://github.com/elastic/elasticsearch/pull/121327) +* System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) +* System data streams are not being upgraded in the feature migration API [#124884](https://github.com/elastic/elasticsearch/pull/124884) (issue: [#124884](https://github.com/elastic/elasticsearch/pull/124884)) Ingest Node: +* Fix geoip databases index access after system feature migration (again) [#122938](https://github.com/elastic/elasticsearch/pull/122938) +* Fix geoip databases index access after system feature migration (take 3) [#124604](https://github.com/elastic/elasticsearch/pull/124604) * apm-data: Use representative count as event.success_count if available [#119995](https://github.com/elastic/elasticsearch/pull/119995) Machine Learning: +* Add `ElasticInferenceServiceCompletionServiceSettings` [#123155](https://github.com/elastic/elasticsearch/pull/123155) +* Add enterprise license check to inference action for semantic text fields [#122293](https://github.com/elastic/elasticsearch/pull/122293) +* Avoid potentially throwing calls to Task#getDescription in model download [#124527](https://github.com/elastic/elasticsearch/pull/124527) * Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` [#125023](https://github.com/elastic/elasticsearch/pull/125023) +* Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) +* Migrate `model_version` to `model_id` when parsing persistent elser inference endpoints [#124769](https://github.com/elastic/elasticsearch/pull/124769) (issue: [#124769](https://github.com/elastic/elasticsearch/pull/124769)) * Provide model size statistics as soon as an anomaly detection job is opened [#124638](https://github.com/elastic/elasticsearch/pull/124638) (issue: [#124638](https://github.com/elastic/elasticsearch/pull/124638)) +* Retry on streaming errors [#123076](https://github.com/elastic/elasticsearch/pull/123076) * Return a Conflict status code if the model deployment is stopped by a user [#125204](https://github.com/elastic/elasticsearch/pull/125204) (issue: [#125204](https://github.com/elastic/elasticsearch/pull/125204)) +* Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) * Set default similarity for Cohere model to cosine [#125370](https://github.com/elastic/elasticsearch/pull/125370) (issue: [#125370](https://github.com/elastic/elasticsearch/pull/125370)) * Updates to allow using Cohere binary embedding response in semantic search queries [#121827](https://github.com/elastic/elasticsearch/pull/121827) +* [Inference API] Fix output stream ordering in `InferenceActionProxy` [#124225](https://github.com/elastic/elasticsearch/pull/124225) Mapping: +* Avoid serializing empty `_source` fields in mappings [#122606](https://github.com/elastic/elasticsearch/pull/122606) * Fix Semantic Text 8.x Upgrade Bug [#125446](https://github.com/elastic/elasticsearch/pull/125446) +* Merge template mappings properly during validation [#124784](https://github.com/elastic/elasticsearch/pull/124784) (issue: [#124784](https://github.com/elastic/elasticsearch/pull/124784)) Ranking: +* Fix LTR query feature with phrases (and two-phase) queries [#125103](https://github.com/elastic/elasticsearch/pull/125103) * Restore `TextSimilarityRankBuilder` XContent output [#124564](https://github.com/elastic/elasticsearch/pull/124564) Relevance: * Prevent Query Rule Creation with Invalid Numeric Match Criteria [#122823](https://github.com/elastic/elasticsearch/pull/122823) Search: +* Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions [#123890](https://github.com/elastic/elasticsearch/pull/123890) +* Don't generate stacktrace in `TaskCancelledException` [#125002](https://github.com/elastic/elasticsearch/pull/125002) +* Fix concurrency issue in `ScriptSortBuilder` [#123757](https://github.com/elastic/elasticsearch/pull/123757) * Fix handling of auto expand replicas for stateless indices [#122365](https://github.com/elastic/elasticsearch/pull/122365) * Handle long overflow in dates [#124048](https://github.com/elastic/elasticsearch/pull/124048) (issue: [#124048](https://github.com/elastic/elasticsearch/pull/124048)) +* Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: [#122357](https://github.com/elastic/elasticsearch/pull/122357)) +* Let MLTQuery throw IAE when no analyzer is set [#124662](https://github.com/elastic/elasticsearch/pull/124662) (issue: [#124662](https://github.com/elastic/elasticsearch/pull/124662)) * Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` [#125650](https://github.com/elastic/elasticsearch/pull/125650) (issue: [#125650](https://github.com/elastic/elasticsearch/pull/125650)) * Minor-Fixes Support 7x segments as archive in 8x / 9x [#125666](https://github.com/elastic/elasticsearch/pull/125666) * Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. [#119503](https://github.com/elastic/elasticsearch/pull/119503) (issue: [#119503](https://github.com/elastic/elasticsearch/pull/119503)) Snapshot/Restore: +* Fork post-snapshot-delete cleanup off master thread [#122731](https://github.com/elastic/elasticsearch/pull/122731) * Limit number of suppressed S3 deletion errors [#123630](https://github.com/elastic/elasticsearch/pull/123630) (issue: [#123630](https://github.com/elastic/elasticsearch/pull/123630)) +* This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. [#124931](https://github.com/elastic/elasticsearch/pull/124931) Suggesters: +* Return an empty suggestion when suggest phase times out [#122575](https://github.com/elastic/elasticsearch/pull/122575) (issue: [#122575](https://github.com/elastic/elasticsearch/pull/122575)) * Support duplicate suggestions in completion field [#121324](https://github.com/elastic/elasticsearch/pull/121324) (issue: [#121324](https://github.com/elastic/elasticsearch/pull/121324)) +Transform: +* If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias [#122074](https://github.com/elastic/elasticsearch/pull/122074) (issue: [#122074](https://github.com/elastic/elasticsearch/pull/122074)) + Vector Search: +* Knn vector rescoring to sort score docs [#122653](https://github.com/elastic/elasticsearch/pull/122653) (issue: [#122653](https://github.com/elastic/elasticsearch/pull/122653)) * Return appropriate error on null dims update instead of npe [#125716](https://github.com/elastic/elasticsearch/pull/125716) - From 674b44134c946919d4bbf8130a42f0c0ee656347 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 1 Apr 2025 16:48:12 +0000 Subject: [PATCH 12/34] [CI] Auto commit changes from spotless --- .../gradle/internal/release/BundleChangelogsTask.java | 2 +- .../gradle/internal/release/ReleaseNotesGenerator.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index 77b41adf7d2a1..51f4c29163a6e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -10,10 +10,10 @@ package org.elasticsearch.gradle.internal.release; import com.fasterxml.jackson.annotation.JsonInclude; - import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; + import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.Directory; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java index 82aa678491abb..5979c654075c9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.TreeMap; import static java.util.Comparator.comparing; From 72e53131fcc3d74d08816e52e5dac4e66b4c89d4 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 7 Apr 2025 12:46:03 -0400 Subject: [PATCH 13/34] Generate breaking-changes and deprecations as well --- .../release/BundleChangelogsTask.java | 2 +- .../release/GenerateReleaseNotesTask.java | 2 + .../resources/templates/breaking-changes.md | 9 +- .../main/resources/templates/deprecations.md | 9 +- docs/release-notes/breaking-changes.md | 7 + .../release-notes/changelog-bundles/9.0.0.yml | 471 +++--------------- docs/release-notes/deprecations.md | 12 +- docs/release-notes/index.md | 45 +- 8 files changed, 135 insertions(+), 422 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index 77b41adf7d2a1..14bd20a1e328b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -74,7 +74,7 @@ public BundleChangelogsTask(ObjectFactory objectFactory, ExecOperations execOper public void executeTask() throws IOException { final String upstreamRemote = gitWrapper.getUpstream(); - String ref = "main"; + String ref = "43827b943938345d348826f5766dd8e986ec7b20"; try { checkoutChangelogs(gitWrapper, upstreamRemote, ref); Properties props = new Properties(); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index a7aa2375d0a79..e00491083dfc8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -111,6 +111,8 @@ public void executeTask() throws IOException { LOGGER.info("Generating release notes..."); ReleaseNotesGenerator.update(this.releaseNotesTemplate.get().getAsFile(), this.releaseNotesFile.get().getAsFile(), bundles); + ReleaseNotesGenerator.update(this.breakingChangesTemplate.get().getAsFile(), this.breakingChangesFile.get().getAsFile(), bundles); + ReleaseNotesGenerator.update(this.deprecationsTemplate.get().getAsFile(), this.deprecationsFile.get().getAsFile(), bundles); // Only update breaking changes and deprecations for new minors // if (qualifiedVersion.revision() == 0) { diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.md b/build-tools-internal/src/main/resources/templates/breaking-changes.md index aec63d1535eec..9f0dc982c0afa 100644 --- a/build-tools-internal/src/main/resources/templates/breaking-changes.md +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.md @@ -13,7 +13,13 @@ If you are migrating from a version prior to version 9.0, you must first upgrade To learn how to upgrade, check out . % ## Next version [elasticsearch-nextversion-breaking-changes] - +<% +for(bundle in changelogBundles) { + def version = bundle.version + def versionWithoutSeparator = bundle.versionWithoutSeparator + def changelogsByTypeByArea = bundle.changelogsByTypeByArea + def unqualifiedVersion = bundle.unqualifiedVersion +%> ## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-breaking-changes] <% if (!changelogsByTypeByArea['breaking']) { @@ -35,3 +41,4 @@ To learn how to upgrade, check out . print "\n\n" } +} diff --git a/build-tools-internal/src/main/resources/templates/deprecations.md b/build-tools-internal/src/main/resources/templates/deprecations.md index 55d4b8bd6d4a0..4470d1f5a78f0 100644 --- a/build-tools-internal/src/main/resources/templates/deprecations.md +++ b/build-tools-internal/src/main/resources/templates/deprecations.md @@ -15,7 +15,13 @@ To give you insight into what deprecated features you’re using, {{es}}: * [Provides a deprecation info API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-migration-deprecations) that scans a cluster’s configuration and mappings for deprecated functionality. % ## Next version [elasticsearch-nextversion-deprecations] - +<% +for(bundle in changelogBundles) { + def version = bundle.version + def versionWithoutSeparator = bundle.versionWithoutSeparator + def changelogsByTypeByArea = bundle.changelogsByTypeByArea + def unqualifiedVersion = bundle.unqualifiedVersion +%> ## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-deprecations] <% if (!changelogsByTypeByArea['deprecation']) { @@ -36,3 +42,4 @@ To give you insight into what deprecated features you’re using, {{es}}: } print "\n\n" } +} diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index b40c1acc7d7d9..56c56935d8962 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -14,6 +14,10 @@ To learn how to upgrade, check out . % ## Next version [elasticsearch-nextversion-breaking-changes] +## 9.0.0 [elasticsearch-900-breaking-changes] + +No breaking changes in this version. + ## 9.1.0 [elasticsearch-910-breaking-changes] Discovery-Plugins: @@ -22,4 +26,7 @@ Discovery-Plugins: Infra/Logging: * Rename deprecation index template [#125606](https://github.com/elastic/elasticsearch/pull/125606) (issue: {es-issue}125445[#125445]) +TLS: +* Drop `TLS_RSA` cipher support for JDK 24 [#123600](https://github.com/elastic/elasticsearch/pull/123600) + diff --git a/docs/release-notes/changelog-bundles/9.0.0.yml b/docs/release-notes/changelog-bundles/9.0.0.yml index 8cfe6dbfc17db..e166b8a9a9964 100644 --- a/docs/release-notes/changelog-bundles/9.0.0.yml +++ b/docs/release-notes/changelog-bundles/9.0.0.yml @@ -1,6 +1,12 @@ version: 9.0.0 -generated: 2025-03-31T18:34:33.370656Z +generated: 2025-04-07T16:44:22.329753Z changelogs: + - pr: 106953 + summary: Optimize usage calculation in ILM policies retrieval API + area: ILM+SLM + type: enhancement + issues: + - 105773 - pr: 113757 summary: Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. area: Mapping @@ -22,11 +28,6 @@ changelogs: area: Mapping type: enhancement issues: [] - - pr: 119886 - summary: Initial support for unmapped fields - area: ES|QL - type: feature - issues: [] - pr: 119995 summary: "apm-data: Use representative count as event.success_count if available" area: Ingest Node @@ -93,45 +94,16 @@ changelogs: area: ILM+SLM type: enhancement issues: [] - - pr: 121392 - summary: Include data streams when converting an existing resource to a system resource - area: Infra/Core - type: bug - issues: [] - pr: 121548 summary: Adding support for specifying embedding type to Jina AI service settings area: Machine Learning type: enhancement issues: [] - - pr: 121667 - summary: Add deprecation warning for flush API - area: Machine Learning - type: deprecation - issues: - - 121506 - deprecation: - area: REST API - title: Add deprecation warning for flush API - details: "The anomaly detection job flush API is deprecated since it is only required for the post data API, which was deprecated since 7.11.0." - impact: "This should have a minimal impact on users as the flush API is only required for the post data API, which was deprecated since 7.11.0." - notable: false - essSettingChange: false - pr: 121784 summary: Optionally allow text similarity reranking to fail area: Search type: enhancement issues: [] - - pr: 121787 - summary: Added optional parameters to QSTR ES|QL function - area: Search - type: feature - issues: - - 120933 - - pr: 121793 - summary: ES|QL - Add scoring for full text functions disjunctions - area: ES|QL - type: enhancement - issues: [] - pr: 121805 summary: Support subset of metrics in aggregate metric double area: ES|QL @@ -142,6 +114,11 @@ changelogs: area: Machine Learning type: bug issues: [] + - pr: 121885 + summary: Introduce batched query execution and data-node side reduce + area: Search + type: enhancement + issues: [] - pr: 121920 summary: Account for the `SearchHit` source in circuit breaker area: Search @@ -158,11 +135,6 @@ changelogs: area: ES|QL type: feature issues: [] - - pr: 121971 - summary: Do not fetch reserved roles from native store when Get Role API is called - area: Authorization - type: enhancement - issues: [] - pr: 122062 summary: Upgrade `discovery-ec2` to AWS SDK v2 area: Discovery-Plugins @@ -209,12 +181,6 @@ changelogs: area: Machine Learning type: enhancement issues: [] - - pr: 122074 - summary: "If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias" - area: Transform - type: bug - issues: - - 121913 - pr: 122134 summary: Adding integration for VoyageAI embeddings and rerank models area: Machine Learning @@ -237,42 +203,16 @@ changelogs: type: enhancement issues: - 121739 - - pr: 122257 - summary: Revive inlinestats - area: ES|QL - type: bug - issues: [] - pr: 122272 summary: "[Inference API] Rename `model_id` prop to model in EIS sparse inference request body" area: Inference type: enhancement issues: [] - - pr: 122278 - summary: Fix serialising the inference update request - area: Machine Learning - type: bug - issues: [] - pr: 122280 summary: Use `FallbackSyntheticSourceBlockLoader` for number fields area: Mapping type: enhancement issues: [] - - pr: 122293 - summary: Add enterprise license check to inference action for semantic text fields - area: Machine Learning - type: bug - issues: [] - - pr: 122326 - summary: System Index Migration Failure Results in a Non-Recoverable State - area: Infra/Core - type: bug - issues: [] - - pr: 122357 - summary: Handle search timeout in `SuggestPhase` - area: Search - type: bug - issues: - - 122186 - pr: 122365 summary: Fix handling of auto expand replicas for stateless indices area: Search @@ -289,22 +229,6 @@ changelogs: area: Health type: enhancement issues: [] - - pr: 122409 - summary: Allow setting the `type` in the reroute processor - area: Ingest Node - type: enhancement - issues: - - 121553 - - pr: 122427 - summary: Improve size limiting string message - area: Infra/Core - type: enhancement - issues: [] - - pr: 122431 - summary: Upgrade AWS SDK to v1.12.746 - area: Snapshot/Restore - type: upgrade - issues: [] - pr: 122458 summary: '`DesiredBalanceReconciler` always returns `AllocationStats`' area: Allocation @@ -320,29 +244,12 @@ changelogs: area: Data streams type: enhancement issues: [] - - pr: 122575 - summary: Return an empty suggestion when suggest phase times out - area: Suggesters - type: bug - issues: - - 122548 - pr: 122586 summary: "ESQL: Fix inconsistent results in using scaled_float field" area: ES|QL type: bug issues: - 122547 - - pr: 122601 - summary: Implicit numeric casting for CASE/GREATEST/LEAST - area: ES|QL - type: bug - issues: - - 121890 - - pr: 122606 - summary: Avoid serializing empty `_source` fields in mappings - area: Mapping - type: bug - issues: [] - pr: 122615 summary: Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. area: Mapping @@ -360,12 +267,6 @@ changelogs: type: enhancement issues: - 117856 - - pr: 122653 - summary: Knn vector rescoring to sort score docs - area: Vector Search - type: bug - issues: - - 119711 - pr: 122660 summary: Render `aggregate_metric_double` area: ES|QL @@ -376,17 +277,6 @@ changelogs: area: ES|QL type: enhancement issues: [] - - pr: 122731 - summary: Fork post-snapshot-delete cleanup off master thread - area: Snapshot/Restore - type: bug - issues: [] - - pr: 122762 - summary: "ESQL: Remove estimated row size assertion" - area: ES|QL - type: bug - issues: - - 121535 - pr: 122821 summary: Fix functions emitting warnings with no source area: ES|QL @@ -418,12 +308,6 @@ changelogs: area: ILM+SLM type: enhancement issues: [] - - pr: 122886 - summary: Add support to VALUES aggregation for spatial types - area: ES|QL - type: bug - issues: - - 122413 - pr: 122890 summary: Introduce `allow_partial_results` setting in ES|QL area: ES|QL @@ -434,11 +318,6 @@ changelogs: area: ES|QL type: enhancement issues: [] - - pr: 122905 - summary: Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover - area: Data streams - type: bug - issues: [] - pr: 122921 summary: Run `TransportGetMappingsAction` on local node area: Indices APIs @@ -449,28 +328,6 @@ changelogs: area: Stats type: enhancement issues: [] - - pr: 122938 - summary: Fix geoip databases index access after system feature migration (again) - area: Ingest Node - type: bug - issues: [] - - pr: 122951 - summary: Updates the deprecation info API to not warn about system indices and data streams - area: Indices APIs - type: bug - issues: [] - - pr: 122960 - summary: Deprecate Behavioral Analytics CRUD apis - area: Search - type: deprecation - issues: [] - deprecation: - area: Search - title: Deprecate Behavioral Analytics CRUD apis - details: "Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. The APIs will still work for now, but will emit warning headers that the API has been deprecated." - impact: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. - notable: false - essSettingChange: false - pr: 122991 summary: "GCS blob store: add `OperationPurpose/Operation` stats counters" area: Snapshot/Restore @@ -481,17 +338,6 @@ changelogs: area: Mapping type: enhancement issues: [] - - pr: 123010 - summary: Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` - area: Engine - type: bug - issues: - - 122974 - - pr: 123076 - summary: Retry on streaming errors - area: Machine Learning - type: bug - issues: [] - pr: 123085 summary: Remove duplicated nested commands area: ES|QL @@ -502,65 +348,22 @@ changelogs: area: Machine Learning type: feature issues: [] - - pr: 123155 - summary: Add `ElasticInferenceServiceCompletionServiceSettings` - area: Machine Learning - type: bug - issues: [] - pr: 123187 summary: Add bit vector support to semantic text area: Vector Search type: enhancement issues: [] - - pr: 123197 - summary: Fix early termination in `LuceneSourceOperator` - area: ES|QL - type: bug - issues: [] - - pr: 123272 - summary: Set Connect Timeout to 5s - area: Machine Learning - type: bug - issues: [] - - pr: 123290 - summary: Fix Driver status iterations and `cpuTime` - area: ES|QL - type: enhancement - issues: - - 122967 - - pr: 123296 - summary: Avoid over collecting in Limit or Lucene Operator - area: ES|QL - type: bug - issues: [] - - pr: 123381 - summary: Push down `StartsWith` and `EndsWith` functions to Lucene - area: ES|QL - type: enhancement - issues: - - 123067 - pr: 123396 summary: Add initial grammar and planning for RRF (snapshot) area: ES|QL type: feature issues: [] - - pr: 123427 - summary: Reduce iteration complexity for plan traversal - area: ES|QL - type: bug - issues: [] - pr: 123460 summary: "ES|QL: Support `::date` in inline cast" area: ES|QL type: enhancement issues: - 116746 - - pr: 123492 - summary: Fix function registry concurrency issues on constructor - area: ES|QL - type: bug - issues: - - 123430 - pr: 123543 summary: Enhance memory accounting for document expansion and introduce max document size limit area: CRUD @@ -576,23 +379,6 @@ changelogs: area: Infra/Core type: enhancement issues: [] - - pr: 123589 - summary: Revive some more of inlinestats functionality - area: ES|QL - type: bug - issues: [] - - pr: 123600 - summary: Drop `TLS_RSA` cipher support for JDK 24 - area: TLS - type: breaking - issues: [] - breaking: - area: Cluster and node setting - title: Drop `TLS_RSA` cipher support for JDK 24 - details: "This change removes `TLS_RSA` ciphers from the list of default supported ciphers, for Elasticsearch deployments running on JDK 24." - impact: "The dropped ciphers are `TLS_RSA_WITH_AES_256_GCM_SHA384`, `TLS_RSA_WITH_AES_128_GCM_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA`, and `TLS_RSA_WITH_AES_128_CBC_SHA`. TLS connections to Elasticsearch using these ciphers will no longer work. Please configure your clients to use one of supported cipher suites." - notable: false - essSettingChange: false - pr: 123630 summary: Limit number of suppressed S3 deletion errors area: Snapshot/Restore @@ -604,26 +390,6 @@ changelogs: area: ILM+SLM type: enhancement issues: [] - - pr: 123728 - summary: Allow skip shards with `_tier` and `_index` in ES|QL - area: ES|QL - type: enhancement - issues: [] - - pr: 123743 - summary: Adjust exception thrown when unable to load hunspell dict - area: Analysis - type: bug - issues: [] - - pr: 123757 - summary: Fix concurrency issue in `ScriptSortBuilder` - area: Search - type: bug - issues: [] - - pr: 123761 - summary: Have create index return a bad request on poor formatting - area: Infra/Core - type: bug - issues: [] - pr: 123763 summary: Skip semantic_text embedding generation when no content is provided. area: Relevance @@ -634,17 +400,6 @@ changelogs: area: Snapshot/Restore type: enhancement issues: [] - - pr: 123890 - summary: Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions - area: Search - type: bug - issues: [] - - pr: 124001 - summary: Use a must boolean statement when pushing down to Lucene when scoring is also needed - area: ES|QL - type: bug - issues: - - 123967 - pr: 124025 summary: "[Inference API] Propagate product use case http header to EIS" area: Machine Learning @@ -661,94 +416,33 @@ changelogs: area: Mapping type: enhancement issues: [] - - pr: 124062 - summary: Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0 - area: Snapshot/Restore - type: upgrade - issues: [] - pr: 124094 summary: ES|QL slow log area: ES|QL type: enhancement issues: [] - - pr: 124107 - summary: Avoid hoarding cluster state references during rollover - area: Indices APIs - type: bug - issues: - - 123893 - - pr: 124149 - summary: Retry ILM async action after reindexing data stream - area: Data streams - type: enhancement - issues: [] - pr: 124177 summary: "Improve error message for ( and [" area: ES|QL type: bug issues: - 124145 - - pr: 124225 - summary: "[Inference API] Fix output stream ordering in `InferenceActionProxy`" - area: Machine Learning - type: bug - issues: [] - pr: 124313 summary: Optimize memory usage in `ShardBulkInferenceActionFilter` area: Search type: enhancement issues: [] - - pr: 124335 - summary: Change the order of the optimization rules - area: ES|QL - type: bug - issues: [] - - pr: 124363 - summary: Set cause on create index request in create from action - area: Data streams - type: enhancement - issues: [] - pr: 124394 summary: Avoid `NamedWritable` in block serialization area: ES|QL type: enhancement issues: [] - - pr: 124424 - summary: Lazy collection copying during node transform - area: ES|QL - type: bug - issues: [] - pr: 124446 summary: "ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate`" area: ES|QL type: bug issues: - 124311 - - pr: 124451 - summary: Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. - area: Downsampling - type: bug - issues: [] - - pr: 124477 - summary: Improve downsample performance by buffering docids and do bulk processing - area: Downsampling - type: enhancement - issues: [] - - pr: 124527 - summary: Avoid potentially throwing calls to Task#getDescription in model download - area: Machine Learning - type: bug - issues: [] - - pr: 124540 - summary: "ES|QL: Fix scoring for full text functions" - area: ES|QL - type: bug - issues: [] - - pr: 124544 - summary: Bump nimbus-jose-jwt to 10.0.2 - area: Security - type: upgrade - issues: [] - pr: 124564 summary: Restore `TextSimilarityRankBuilder` XContent output area: Ranking @@ -774,39 +468,17 @@ changelogs: area: ES|QL type: enhancement issues: [] - - pr: 124604 - summary: Fix geoip databases index access after system feature migration (take 3) - area: Ingest Node - type: bug - issues: [] - pr: 124610 summary: Remove page alignment in exchange sink area: ES|QL type: enhancement issues: [] - - pr: 124611 - summary: Reuse child `outputSet` inside the plan where possible - area: ES|QL - type: enhancement - issues: [] - pr: 124638 summary: Provide model size statistics as soon as an anomaly detection job is opened area: Machine Learning type: bug issues: - 121168 - - pr: 124651 - summary: Fix system data streams to be restorable from a snapshot - area: Infra/Core - type: bug - issues: - - 89261 - - pr: 124662 - summary: Let MLTQuery throw IAE when no analyzer is set - area: Search - type: bug - issues: - - 124562 - pr: 124669 summary: Release semantic_text as a GA feature area: Mapping @@ -834,39 +506,16 @@ changelogs: type: enhancement issues: - 117096 - - pr: 124732 - summary: Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. - area: Infra/Core - type: bug - issues: - - 124667 - pr: 124738 summary: Upgrade AWS v2 SDK to 2.30.38 area: Machine Learning type: upgrade issues: [] - - pr: 124739 - summary: Improve rolling up metrics - area: Downsampling - type: enhancement - issues: [] - - pr: 124769 - summary: Migrate `model_version` to `model_id` when parsing persistent elser inference endpoints - area: Machine Learning - type: bug - issues: - - 124675 - pr: 124782 summary: "Aggs: Let terms run in global ords mode no match" area: Aggregations type: bug issues: [] - - pr: 124784 - summary: Merge template mappings properly during validation - area: Mapping - type: bug - issues: - - 123372 - pr: 124823 summary: Report failures on partial results area: ES|QL @@ -877,27 +526,6 @@ changelogs: area: Distributed type: bug issues: [] - - pr: 124843 - summary: Ignore _JAVA_OPTIONS - area: Infra/CLI - type: enhancement - issues: [] - - pr: 124865 - summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." - area: Infra/Core - type: upgrade - issues: [] - - pr: 124873 - summary: Restore V8 REST compatibility around highlight `force_source` parameter - area: Highlighting - type: bug - issues: [] - - pr: 124884 - summary: System data streams are not being upgraded in the feature migration API - area: Infra/Core - type: bug - issues: - - 122949 - pr: 124898 summary: Add cache support in `TransportGetAllocationStatsAction` area: Allocation @@ -929,21 +557,6 @@ changelogs: area: ES|QL type: enhancement issues: [] - - pr: 124931 - summary: This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. - area: Snapshot/Restore - type: bug - issues: [] - - pr: 124936 - summary: Indicate when errors represent timeouts - area: Infra/REST API - type: enhancement - issues: [] - - pr: 125002 - summary: Don't generate stacktrace in `TaskCancelledException` - area: Search - type: bug - issues: [] - pr: 125023 summary: Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` area: Machine Learning @@ -955,26 +568,21 @@ changelogs: type: enhancement issues: - 124181 - - pr: 125103 - summary: Fix LTR query feature with phrases (and two-phase) queries - area: Ranking - type: bug - issues: [] - pr: 125117 summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." area: Infra/Core type: upgrade issues: [] - - pr: 125159 - summary: Update bundled JDK to Java 24 - area: Packaging - type: upgrade - issues: [] - pr: 125171 summary: Reindex data stream indices on different nodes area: Data streams type: enhancement issues: [] + - pr: 125191 + summary: Fix sorting when `aggregate_metric_double` present + area: ES|QL + type: enhancement + issues: [] - pr: 125204 summary: Return a Conflict status code if the model deployment is stopped by a user area: Machine Learning @@ -1040,6 +648,18 @@ changelogs: area: Mapping type: bug issues: [] + - pr: 125477 + summary: Prevent get datafeeds stats API returning an error when local tasks are slow to stop + area: Machine Learning + type: bug + issues: + - 104160 + - pr: 125520 + summary: Add `FailedShardEntry` info to shard-failed task source string + area: Allocation + type: enhancement + issues: + - 102606 - pr: 125529 summary: Store arrays offsets for boolean fields natively with synthetic source area: Mapping @@ -1090,6 +710,11 @@ changelogs: area: Search type: bug issues: [] + - pr: 125690 + summary: Take double parameter markers for identifiers out of snapshot + area: ES|QL + type: enhancement + issues: [] - pr: 125709 summary: Store arrays offsets for unsigned long fields natively with synthetic source area: Mapping @@ -1100,3 +725,31 @@ changelogs: area: Vector Search type: bug issues: [] + - pr: 125764 + summary: Fix `ReplaceMissingFieldsWithNull` + area: ES|QL + type: bug + issues: + - 126036 + - 121754 + - 126030 + - pr: 125881 + summary: Fixes a invalid warning from being issued when restoring a system data stream from a snapshot. + area: Data streams + type: bug + issues: [] + - pr: 125916 + summary: Re-enable parallel collection for field sorted top hits + area: Search + type: bug + issues: [] + - pr: 125930 + summary: Infer the score mode to use from the Lucene collector + area: ES|QL + type: enhancement + issues: [] + - pr: 126077 + summary: Preventing `ConcurrentModificationException` when updating settings for more than one index + area: Indices APIs + type: bug + issues: [] diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index 5c30b27916d91..4ad739e6749bb 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -16,6 +16,16 @@ To give you insight into what deprecated features you’re using, {{es}}: % ## Next version [elasticsearch-nextversion-deprecations] -## 9.1.0 [elasticsearch-910-deprecations] +## 9.0.0 [elasticsearch-900-deprecations] No deprecations in this version. + +## 9.1.0 [elasticsearch-910-deprecations] + +Machine Learning: +* Add deprecation warning for flush API [#121667](https://github.com/elastic/elasticsearch/pull/121667) (issue: {es-issue}121506[#121506]) + +Search: +* Deprecate Behavioral Analytics CRUD apis [#122960](https://github.com/elastic/elasticsearch/pull/122960) + + diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index f819f2df12c8f..b2372f5d9da68 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -21,17 +21,45 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * -## 9.1.0 [elasticsearch-910-release-notes] +## 9.0.0 [elasticsearch-900-release-notes] -### Fixes [elasticsearch-910-fixes] +### Features and enhancements [elasticsearch-900-features-enhancements] + +Allocation: +* Add `FailedShardEntry` info to shard-failed task source string [#125520](https://github.com/elastic/elasticsearch/pull/125520) (issue: [#125520](https://github.com/elastic/elasticsearch/pull/125520)) ES|QL: -* Catch parsing exception [#124958](https://github.com/elastic/elasticsearch/pull/124958) (issue: [#124958](https://github.com/elastic/elasticsearch/pull/124958)) +* Fix sorting when `aggregate_metric_double` present [#125191](https://github.com/elastic/elasticsearch/pull/125191) +* Infer the score mode to use from the Lucene collector [#125930](https://github.com/elastic/elasticsearch/pull/125930) +* Take double parameter markers for identifiers out of snapshot [#125690](https://github.com/elastic/elasticsearch/pull/125690) +ILM+SLM: +* Optimize usage calculation in ILM policies retrieval API [#106953](https://github.com/elastic/elasticsearch/pull/106953) (issue: [#106953](https://github.com/elastic/elasticsearch/pull/106953)) -## 9.0.0 [elasticsearch-900-release-notes] +Search: +* Introduce batched query execution and data-node side reduce [#121885](https://github.com/elastic/elasticsearch/pull/121885) + +### Fixes [elasticsearch-900-fixes] -### Highlights [elasticsearch-900-highlights] +Data streams: +* Fixes a invalid warning from being issued when restoring a system data stream from a snapshot. [#125881](https://github.com/elastic/elasticsearch/pull/125881) + +ES|QL: +* Fix `ReplaceMissingFieldsWithNull` [#125764](https://github.com/elastic/elasticsearch/pull/125764) (issues: [#125764](https://github.com/elastic/elasticsearch/pull/125764), [#125764](https://github.com/elastic/elasticsearch/pull/125764), [#125764](https://github.com/elastic/elasticsearch/pull/125764)) + +Indices APIs: +* Preventing `ConcurrentModificationException` when updating settings for more than one index [#126077](https://github.com/elastic/elasticsearch/pull/126077) + +Machine Learning: +* Prevent get datafeeds stats API returning an error when local tasks are slow to stop [#125477](https://github.com/elastic/elasticsearch/pull/125477) (issue: [#125477](https://github.com/elastic/elasticsearch/pull/125477)) + +Search: +* Re-enable parallel collection for field sorted top hits [#125916](https://github.com/elastic/elasticsearch/pull/125916) + + +## 9.1.0 [elasticsearch-910-release-notes] + +### Highlights [elasticsearch-910-highlights] ::::{dropdown} Release semantic_text as a GA feature semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. @@ -39,7 +67,7 @@ semantic_text is now an official GA (generally available) feature! This field ty For more information, check [PR #124669](https://github.com/elastic/elasticsearch/pull/124669). :::: -### Features and enhancements [elasticsearch-900-features-enhancements] +### Features and enhancements [elasticsearch-910-features-enhancements] Allocation: * Add cache support in `TransportGetAllocationStatsAction` [#124898](https://github.com/elastic/elasticsearch/pull/124898) (issue: [#124898](https://github.com/elastic/elasticsearch/pull/124898)) @@ -120,7 +148,6 @@ Infra/CLI: Infra/Core: * Give Kibana user 'all' permissions for .entity_analytics.* indices [#123588](https://github.com/elastic/elasticsearch/pull/123588) * Improve size limiting string message [#122427](https://github.com/elastic/elasticsearch/pull/122427) -* Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#124865](https://github.com/elastic/elasticsearch/pull/124865) * Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#125117](https://github.com/elastic/elasticsearch/pull/125117) Infra/REST API: @@ -205,7 +232,7 @@ Vector Search: Watcher: * Run `TransportGetWatcherSettingsAction` on local node [#122857](https://github.com/elastic/elasticsearch/pull/122857) -### Fixes [elasticsearch-900-fixes] +### Fixes [elasticsearch-910-fixes] Aggregations: * Aggs: Let terms run in global ords mode no match [#124782](https://github.com/elastic/elasticsearch/pull/124782) @@ -235,6 +262,7 @@ EQL: ES|QL: * Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: [#122886](https://github.com/elastic/elasticsearch/pull/122886)) * Avoid over collecting in Limit or Lucene Operator [#123296](https://github.com/elastic/elasticsearch/pull/123296) +* Catch parsing exception [#124958](https://github.com/elastic/elasticsearch/pull/124958) (issue: [#124958](https://github.com/elastic/elasticsearch/pull/124958)) * Change the order of the optimization rules [#124335](https://github.com/elastic/elasticsearch/pull/124335) * ESQL - date nanos range bug? [#125345](https://github.com/elastic/elasticsearch/pull/125345) (issue: [#125345](https://github.com/elastic/elasticsearch/pull/125345)) * ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate` [#124446](https://github.com/elastic/elasticsearch/pull/124446) (issue: [#124446](https://github.com/elastic/elasticsearch/pull/124446)) @@ -274,7 +302,6 @@ Infra/Core: * Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. [#124732](https://github.com/elastic/elasticsearch/pull/124732) (issue: [#124732](https://github.com/elastic/elasticsearch/pull/124732)) * Reduce Data Loss in System Indices Migration [#121327](https://github.com/elastic/elasticsearch/pull/121327) * System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) -* System data streams are not being upgraded in the feature migration API [#124884](https://github.com/elastic/elasticsearch/pull/124884) (issue: [#124884](https://github.com/elastic/elasticsearch/pull/124884)) Ingest Node: * Fix geoip databases index access after system feature migration (again) [#122938](https://github.com/elastic/elasticsearch/pull/122938) From 4585378ecea5b52ba18c92396eeacd30a1b40b41 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 7 Apr 2025 13:46:05 -0400 Subject: [PATCH 14/34] Fixing 9.0.0 release notes --- docs/release-notes/breaking-changes.md | 87 +- .../release-notes/changelog-bundles/9.0.0.yml | 2410 ++++++++++++++++- docs/release-notes/deprecations.md | 18 +- docs/release-notes/index.md | 421 +++ 4 files changed, 2915 insertions(+), 21 deletions(-) diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index 56c56935d8962..e71e68a74777b 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -16,7 +16,92 @@ To learn how to upgrade, check out . ## 9.0.0 [elasticsearch-900-breaking-changes] -No breaking changes in this version. +Aggregations: +* Remove date histogram boolean support [#118484](https://github.com/elastic/elasticsearch/pull/118484) + +Allocation: +* Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831) +* Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting [#114207](https://github.com/elastic/elasticsearch/pull/114207) +* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: {es-issue}88978[#88978]) + +Analysis: +* Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146) +* The 'german2' stemmer is now an alias for the 'german' snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614) +* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: {es-issue}113050[#113050]) +* The Korean dictionary for Nori has been updated [#114124](https://github.com/elastic/elasticsearch/pull/114124) + +Authentication: +* Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password +will prevent node from starting [#118366](https://github.com/elastic/elasticsearch/pull/118366) + +Cluster Coordination: +* Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903) + +EQL: +* Set allow_partial_search_results=true by default [#120267](https://github.com/elastic/elasticsearch/pull/120267) + +Extract&Transform: +* Restrict Connector APIs to manage/monitor_connector privileges [#119863](https://github.com/elastic/elasticsearch/pull/119863) + +Highlighting: +* Remove support for deprecated `force_source` highlighting parameter [#116943](https://github.com/elastic/elasticsearch/pull/116943) + +Indices APIs: +* Apply more strict parsing of actions in bulk API [#115923](https://github.com/elastic/elasticsearch/pull/115923) +* Remove deprecated local attribute from alias APIs [#115393](https://github.com/elastic/elasticsearch/pull/115393) +* Remove the ability to read frozen indices [#120108](https://github.com/elastic/elasticsearch/pull/120108) +* Remove unfreeze REST endpoint [#119227](https://github.com/elastic/elasticsearch/pull/119227) + +Infra/Core: +* Change Elasticsearch timeouts to 429 response instead of 5xx [#116026](https://github.com/elastic/elasticsearch/pull/116026) +* Limit `ByteSizeUnit` to 2 decimals [#120142](https://github.com/elastic/elasticsearch/pull/120142) +* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: {es-issue}104574[#104574]) +* Remove any references to org.elasticsearch.core.RestApiVersion#V_7 [#118103](https://github.com/elastic/elasticsearch/pull/118103) + +Infra/Logging: +* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: {es-issue}83251[#83251]) + +Infra/Metrics: +* Deprecated tracing.apm.* settings got removed. [#119926](https://github.com/elastic/elasticsearch/pull/119926) + +Infra/REST API: +* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: {es-issue}89387[#89387]) + +Ingest Node: +* Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077) +* Remove ignored fallback option on GeoIP processor [#116112](https://github.com/elastic/elasticsearch/pull/116112) + +Logs: +* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: {es-issue}106489[#106489]) + +Machine Learning: +* Disable machine learning on macOS x86_64 [#104125](https://github.com/elastic/elasticsearch/pull/104125) + +Mapping: +* Remove support for type, fields, `copy_to` and boost in metadata field definition [#118825](https://github.com/elastic/elasticsearch/pull/118825) +* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: {es-issue}118596[#118596]) + +Search: +* Adjust `random_score` default field to `_seq_no` field [#118671](https://github.com/elastic/elasticsearch/pull/118671) +* Change Semantic Text To Act Like A Normal Text Field [#120813](https://github.com/elastic/elasticsearch/pull/120813) +* Remove legacy params from range query [#116970](https://github.com/elastic/elasticsearch/pull/116970) + +Snapshot/Restore: +* Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting [#114202](https://github.com/elastic/elasticsearch/pull/114202) + +TLS: +* Remove TLSv1.1 from default protocols [#121731](https://github.com/elastic/elasticsearch/pull/121731) + +Transform: +* Remove `data_frame_transforms` roles [#117519](https://github.com/elastic/elasticsearch/pull/117519) + +Vector Search: +* Remove old `_knn_search` tech preview API in v9 [#118104](https://github.com/elastic/elasticsearch/pull/118104) + +Watcher: +* Removing support for types field in watcher search [#120748](https://github.com/elastic/elasticsearch/pull/120748) + + ## 9.1.0 [elasticsearch-910-breaking-changes] diff --git a/docs/release-notes/changelog-bundles/9.0.0.yml b/docs/release-notes/changelog-bundles/9.0.0.yml index e166b8a9a9964..efbd24564c697 100644 --- a/docs/release-notes/changelog-bundles/9.0.0.yml +++ b/docs/release-notes/changelog-bundles/9.0.0.yml @@ -1,61 +1,1952 @@ version: 9.0.0 -generated: 2025-04-07T16:44:22.329753Z +generated: 2025-04-07T17:44:21.076873Z changelogs: + - pr: 90529 + summary: Output a consistent format when generating error json + area: Infra/REST API + type: breaking + issues: + - 89387 + breaking: + area: REST API + title: Error JSON structure has changed when detailed errors are disabled + details: |- + This change modifies the JSON format of error messages returned to REST clients + when detailed messages are turned off. + Previously, JSON returned when an exception occurred, and `http.detailed_errors.enabled: false` was set, + just consisted of a single `"error"` text field with some basic information. + Setting `http.detailed_errors.enabled: true` (the default) changed this field + to an object with more detailed information. + With this change, non-detailed errors now have the same structure as detailed errors. `"error"` will now always + be an object with, at a minimum, a `"type"` and `"reason"` field. Additional fields are included when detailed + errors are enabled. + To use the previous structure for non-detailed errors, use the v8 REST API. + impact: |- + If you have set `http.detailed_errors.enabled: false` (the default is `true`) + the structure of JSON when any exceptions occur now matches the structure when + detailed errors are enabled. + To use the previous structure for non-detailed errors, use the v8 REST API. + notable: false + essSettingChange: false + - pr: 104125 + summary: Disable machine learning on macOS x86_64 + area: Machine Learning + type: breaking + issues: [] + breaking: + area: Packaging + title: Disable machine learning on macOS x86_64 + details: "The machine learning plugin is permanently disabled on macOS x86_64. For the last three years Apple has been selling hardware based on the arm64 architecture, and support will increasingly focus on this architecture in the future. Changes to upstream dependencies of Elastic's machine learning functionality have made it unviable for Elastic to continue to build machine learning on macOS x86_64." + impact: "To continue to use machine learning functionality on macOS please switch to an arm64 machine (Apple silicon). Alternatively, it will still be possible to run Elasticsearch with machine learning enabled in a Docker container on macOS x86_64." + notable: false + essSettingChange: false - pr: 106953 summary: Optimize usage calculation in ILM policies retrieval API area: ILM+SLM type: enhancement - issues: - - 105773 - - pr: 113757 - summary: Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. - area: Mapping + issues: + - 105773 + - pr: 111104 + summary: "ESQL: Enable async get to support formatting" + area: ES|QL + type: feature + issues: + - 110926 + - pr: 111494 + summary: Extensible Completion Postings Formats + area: Suggesters + type: enhancement + issues: [] + - pr: 111852 + summary: Add DeBERTa-V2/V3 tokenizer + area: Machine Learning + type: enhancement + issues: [] + - pr: 112258 + summary: Updated Date Range to Follow Documentation When Assuming Missing Values + area: Search + type: bug + issues: + - 111484 + - pr: 112567 + summary: Track shard snapshot progress during node shutdown + area: Snapshot/Restore + type: enhancement + issues: [] + - pr: 112834 + summary: Increase `replica_unassigned_buffer_time` default from 3s to 5s + area: Health + type: enhancement + issues: [] + - pr: 112903 + summary: Remove unsupported legacy value for `discovery.type` + area: Cluster Coordination + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Remove unsupported legacy value for `discovery.type` + details: "Earlier versions of {es} had a `discovery.type` setting which permitted values that referred to legacy discovery types. From v9.0.0 onwards, the only supported values for this setting are `multi-node` (the default) and `single-node`." + impact: Remove any value for `discovery.type` from your `elasticsearch.yml` configuration file except for `multi-node` and `single-node`. + notable: false + essSettingChange: false + - pr: 113102 + summary: Trigger merges after recovery + area: Recovery + type: enhancement + issues: [] + - pr: 113120 + summary: ESQL - enabling scoring with METADATA `_score` + area: ES|QL + type: enhancement + issues: [] + - pr: 113131 + summary: Emit deprecation warning when executing one of the rollup APIs + area: Rollup + type: deprecation + issues: [] + deprecation: + area: Rollup + title: Emit deprecation warning when executing one of the rollup APIs + details: Rollup is already deprecated since 8.11.0 via documentation and since 8.15.0 it is no longer possible to create new rollup jobs in clusters without rollup usage. This change updates the rollup APIs to emit a deprecation warning. + impact: Returning a deprecation warning when using one of the rollup APIs. + notable: false + essSettingChange: false + - pr: 113216 + summary: "[Inference API] Deprecate elser service" + area: Machine Learning + type: deprecation + issues: [] + deprecation: + area: REST API + title: "[Inference API] Deprecate elser service" + details: The `elser` service of the inference API will be removed in an upcoming release. Please use the elasticsearch service instead. + impact: "In the current version there is no impact. In a future version, users of the `elser` service will no longer be able to use it, and will be required to use the `elasticsearch` service to access elser through the inference API." + notable: false + essSettingChange: false + - pr: 113237 + summary: Retry throttled snapshot deletions + area: Snapshot/Restore + type: bug + issues: [] + - pr: 113413 + summary: Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null. + area: Infra/REST API + type: bug + issues: + - 113413 + - pr: 113425 + summary: Add `ensureGreen` test method for use with `adminClient` + area: Infra/Metrics + type: enhancement + issues: [] + - pr: 113462 + summary: Suppress merge-on-recovery for older indices + area: CRUD + type: enhancement + issues: [] + - pr: 113482 + summary: The 'persian' analyzer has stemmer by default + area: Analysis + type: breaking + issues: + - 113050 + breaking: + area: Analysis + title: The 'persian' analyzer has stemmer by default + details: Lucene 10 has added a final stemming step to its PersianAnalyzer that Elasticsearch exposes as 'persian' analyzer. Existing indices will keep the old non-stemming behaviour while new indices will see the updated behaviour with added stemming. Users that wish to maintain the non-stemming behaviour need to define their own analyzer as outlined in https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer. Users that wish to use the new stemming behaviour for existing indices will have to reindex their data. + impact: Indexing with the 'persian' analyzer will produce slightly different tokens. Users should check if this impacts their search results. If they wish to maintain the legacy non-stemming behaviour they can define their own analyzer equivalent as explained in https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer. + notable: false + essSettingChange: false + - pr: 113561 + summary: Add link to Circuit Breaker "Data too large" exception message + area: Infra/Circuit Breakers + type: enhancement + issues: [] + - pr: 113614 + summary: The 'german2' stemmer is now an alias for the 'german' snowball stemmer + area: Analysis + type: breaking + issues: [] + breaking: + area: Analysis + title: The "german2" snowball stemmer is now an alias for the "german" stemmer + details: "Lucene 10 has merged the improved \"german2\" snowball language stemmer with the \"german\" stemmer. For Elasticsearch, \"german2\" is now a deprecated alias for \"german\". This may results in slightly different tokens being generated for terms with umlaut substitution (like \"ue\" for \"ü\" etc...)" + impact: Replace usages of "german2" with "german" in analysis configuration. Old indices that use the "german" stemmer should be reindexed if possible. + notable: false + essSettingChange: false + - pr: 113757 + summary: Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. + area: Mapping + type: enhancement + issues: [] + - pr: 113827 + summary: Add Optional Source Filtering to Source Loaders + area: Mapping + type: enhancement + issues: [] + - pr: 114002 + summary: Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts + area: Infra/Scripting + type: enhancement + issues: [] + - pr: 114124 + summary: The Korean dictionary for Nori has been updated + area: Analysis + type: breaking + issues: [] + breaking: + area: Analysis + title: The Korean dictionary for Nori has been updated + details: Lucene 10 ships with an updated Korean dictionary (mecab-ko-dic-2.1.1). For details see https://github.com/apache/lucene/issues/11452. Users experiencing changes in search behaviour on existing data are advised to reindex. + impact: The change is small and should generally provide better analysis results. Existing indices for full-text use cases should be reindexed though. + notable: false + essSettingChange: false + - pr: 114146 + summary: Snowball stemmers have been upgraded + area: Analysis + type: breaking + issues: [] + breaking: + area: Analysis + title: Snowball stemmers have been upgraded + details: Lucene 10 ships with an upgrade of its Snowball stemmers. For details see https://github.com/apache/lucene/issues/13209. Users using Snowball stemmers that are experiencing changes in search behaviour on existing data are advised to reindex. + impact: "The upgrade should generally provide improved stemming results. Small changes in token analysis can lead to mismatches with previously index data, so existing indices using Snowball stemmers as part of their analysis chain should be reindexed." + notable: false + essSettingChange: false + - pr: 114177 + summary: "Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]`" + area: Infra/Metrics + type: bug + issues: [] + - pr: 114202 + summary: Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting + area: Snapshot/Restore + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting + details: The `xpack.searchable.snapshot.allocate_on_rolling_restart` setting was created as an escape-hatch just in case relying on the `cluster.routing.allocation.enable=primaries` setting for allocating searchable snapshots during rolling restarts had some unintended side-effects. It has been deprecated since 8.2.0. + impact: Remove `xpack.searchable.snapshot.allocate_on_rolling_restart` from your settings if present. + notable: false + essSettingChange: false + - pr: 114207 + summary: Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting + area: Allocation + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting + details: "Prior to 7.8, whenever a cluster had only a single data node, the watermarks would not be respected. In order to change this in 7.8+ in a backwards compatible way, we introduced the `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` node setting. The setting was deprecated in 7.14 and was made to accept only true in 8.0" + impact: No known end user impact + notable: false + essSettingChange: false + - pr: 114231 + summary: Remove cluster state from `/_cluster/reroute` response + area: Allocation + type: breaking + issues: + - 88978 + breaking: + area: REST API + title: Remove cluster state from `/_cluster/reroute` response + details: The `POST /_cluster/reroute` API no longer returns the cluster state in its response. The `?metric` query parameter to this API now has no effect and its use will be forbidden in a future version. + impact: Cease usage of the `?metric` query parameter when calling the `POST /_cluster/reroute` API. + notable: false + essSettingChange: false + - pr: 114303 + summary: Fix TDigestState.read CB leaks + area: ES|QL + type: bug + issues: + - 114194 + - pr: 114317 + summary: "ESQL: CATEGORIZE as a `BlockHash`" + area: ES|QL + type: enhancement + issues: [] + - pr: 114445 + summary: Wrap jackson exception on malformed json string + area: Infra/Core + type: bug + issues: + - 114142 + - pr: 114566 + summary: Use Azure blob batch API to delete blobs in batches + area: Distributed + type: enhancement + issues: [] + - pr: 114618 + summary: Add a new index setting to skip recovery source when synthetic source is enabled + area: Logs + type: enhancement + issues: [] + - pr: 114623 + summary: Preserve thread context when waiting for segment generation in RTG + area: CRUD + type: bug + issues: [] + - pr: 114741 + summary: Upgrade to Lucene 10 + area: Search + type: upgrade + issues: [] + - pr: 114813 + summary: Retry `S3BlobContainer#getRegister` on all exceptions + area: Snapshot/Restore + type: enhancement + issues: [] + - pr: 114837 + summary: Add warning headers for ingest pipelines containing special characters + area: Ingest Node + type: bug + issues: + - 104411 + - pr: 114879 + summary: Add refresh `.security` index call between security migrations + area: Security + type: enhancement + issues: [] + - pr: 114914 + summary: Adding chunking settings to `IbmWatsonxService` + area: Machine Learning + type: enhancement + issues: [] + - pr: 115020 + summary: Adding endpoint creation validation for all task types to remaining services + area: Machine Learning + type: enhancement + issues: [] + - pr: 115241 + summary: "[Security Solution] Add `create_index` to `kibana_system` role for index/DS `.logs-endpoint.action.responses-*`" + area: Authorization + type: enhancement + issues: [] + - pr: 115314 + summary: Only aggregations require at least one shard request + area: Search + type: enhancement + issues: [] + - pr: 115383 + summary: Only publish desired balance gauges on master + area: Allocation + type: enhancement + issues: [] + - pr: 115393 + summary: Remove deprecated local attribute from alias APIs + area: Indices APIs + type: breaking + issues: [] + breaking: + area: REST API + title: Remove deprecated local attribute from alias APIs + details: "The following APIs no longer accept the `?local` query parameter: `GET /_alias`, `GET /_aliases`, `GET /_alias/{name}`, `HEAD /_alias/{name}`, `GET /{index}/_alias`, `HEAD /{index}/_alias`, `GET /{index}/_alias/{name}`, `HEAD /{index}/_alias/{name}`, `GET /_cat/aliases`, and `GET /_cat/aliases/{alias}`. This parameter has been deprecated and ignored since version 8.12." + impact: Cease usage of the `?local` query parameter when calling the listed APIs. + notable: false + essSettingChange: false + - pr: 115616 + summary: Fix double lookup failure on ESQL + area: ES|QL + type: bug + issues: + - 111398 + - pr: 115721 + summary: Change Reindexing metrics unit from millis to seconds + area: Reindex + type: enhancement + issues: [] + - pr: 115779 + summary: Don't allow secure settings in YML config (109115) + area: Infra/Settings + type: bug + issues: + - 109115 + - pr: 115831 + summary: Increase minimum threshold in shard balancer + area: Allocation + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Minimum shard balancer threshold is now 1.0 + details: "Earlier versions of {es} accepted any non-negative value for `cluster.routing.allocation.balance.threshold`, but values smaller than `1.0` do not make sense and have been ignored since version 8.6.1. From 9.0.0 these nonsensical values are now forbidden." + impact: Do not set `cluster.routing.allocation.balance.threshold` to a value less than `1.0`. + notable: false + essSettingChange: false + - pr: 115836 + summary: Catch and handle disconnect exceptions in search + area: Search + type: bug + issues: [] + - pr: 115876 + summary: Inference duration and error metrics + area: Machine Learning + type: enhancement + issues: [] + - pr: 115923 + summary: Apply more strict parsing of actions in bulk API + area: Indices APIs + type: breaking + issues: [] + breaking: + area: REST API + title: Apply more strict parsing of actions in bulk API + details: "Previously, the following classes of malformed input were deprecated but not rejected in the action lines of the a bulk request: missing closing brace; additional keys after the action (which were ignored); additional data after the closing brace (which was ignored). They will now be considered errors and rejected." + impact: Users must provide well-formed input when using the bulk API. (They can request REST API compatibility with v8 to get the previous behaviour back as an interim measure.) + notable: false + essSettingChange: false + - pr: 115930 + summary: Inconsistency in the _analyzer api when the index is not included + area: Search + type: bug + issues: [] + - pr: 115938 + summary: Wait for up to 2 seconds for yellow status before starting search + area: Machine Learning + type: bug + issues: + - 107777 + - 105955 + - 107815 + - 112191 + - pr: 116026 + summary: Change Elasticsearch timeouts to 429 response instead of 5xx + area: Infra/Core + type: breaking + issues: [] + breaking: + area: REST API + title: Change most Elasticsearch timeouts to 429 response instead of 5xx + details: "When a timeout occurs in most REST requests, whether via a per-request timeout, or a system default, the request would return a 5xx response code. The response code from those APIs when a timeout occurs is now 429." + impact: Adjust any code relying on retrying on 5xx responses for timeouts to look for a 429 response code and inspect the response to determine whether a timeout occured. + notable: false + essSettingChange: false + - pr: 116043 + summary: Support partial sort fields in TopN pushdown + area: ES|QL + type: enhancement + issues: + - 114515 + - pr: 116060 + summary: Fix leak in `DfsQueryPhase` and introduce search disconnect stress test + area: Search + type: bug + issues: + - 115056 + - pr: 116077 + summary: Remove `ecs` option on `user_agent` processor + area: Ingest Node + type: breaking + issues: [] + breaking: + area: Ingest + title: Remove `ecs` option on `user_agent` processor + details: The `user_agent` ingest processor no longer accepts the `ecs` option. (It was previously deprecated and ignored.) + impact: Users should stop using the `ecs` option when creating instances of the `user_agent` ingest processor. The option will be removed from existing processors stored in the cluster state on upgrade. + notable: false + essSettingChange: false + - pr: 116112 + summary: Remove ignored fallback option on GeoIP processor + area: Ingest Node + type: breaking + issues: [] + breaking: + area: Ingest + title: Remove ignored fallback option on GeoIP processor + details: The option fallback_to_default_databases on the geoip ingest processor has been removed. (It was deprecated and ignored since 8.0.0.) + impact: Customers should stop remove the noop fallback_to_default_databases option on any geoip ingest processors. + notable: false + essSettingChange: false + - pr: 116115 + summary: Allow http unsafe buffers by default + area: Network + type: enhancement + issues: [] + - pr: 116174 + summary: Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations + area: Aggregations + type: bug + issues: + - 115777 + - pr: 116259 + summary: Fix `_type` deprecation on simulate pipeline API + area: Ingest Node + type: deprecation + issues: [] + deprecation: + area: REST API + title: Document `_type` deprecated on simulate pipeline API + details: "Passing a document with a `_type` property is deprecated in the `/_ingest/pipeline/{id}/_simulate` and `/_ingest/pipeline/_simulate` APIs." + impact: "Users should already have stopped using mapping types, which were deprecated in {es} 7. This deprecation warning will fire if they specify mapping types on documents pass to the simulate pipeline API." + notable: false + essSettingChange: false + - pr: 116388 + summary: Add support for partial shard results + area: EQL + type: enhancement + issues: [] + - pr: 116531 + summary: "Add a standard deviation aggregating function: STD_DEV" + area: ES|QL + type: enhancement + issues: [] + - pr: 116663 + summary: KNN vector rescoring for quantized vectors + area: Vector Search + type: feature + issues: [] + - pr: 116687 + summary: Add LogsDB option to route on sort fields + area: Logs + type: enhancement + issues: [] + - pr: 116692 + summary: Remove all mentions of eis and gateway and deprecate flags that do + area: Machine Learning + type: enhancement + issues: [] + - pr: 116739 + summary: Change default Docker image to be based on UBI minimal instead of Ubuntu + area: Infra/Core + type: enhancement + issues: [] + - pr: 116755 + summary: Smarter field caps with subscribable listener + area: ES|QL + type: enhancement + issues: [] + - pr: 116765 + summary: Metrics for incremental bulk splits + area: Distributed + type: enhancement + issues: [] + - pr: 116868 + summary: Run `TransportGetComponentTemplateAction` on local node + area: Indices APIs + type: enhancement + issues: [] + - pr: 116904 + summary: Add a not-master state for desired balance + area: Allocation + type: enhancement + issues: [] + - pr: 116922 + summary: Always check if index mode is logsdb + area: Logs + type: bug + issues: [] + - pr: 116943 + summary: Remove support for deprecated `force_source` highlighting parameter + area: Highlighting + type: breaking + issues: [] + breaking: + area: REST API + title: Remove support for deprecated `force_source` highlighting parameter + details: The deprecated highlighting `force_source` parameter is no longer supported. + impact: Users should remove usages of the `force_source` parameter from their search requests. + notable: false + essSettingChange: false + - pr: 116964 + summary: "Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions" + area: ES|QL + type: feature + issues: + - 104875 + - pr: 116970 + summary: Remove legacy params from range query + area: Search + type: breaking + issues: [] + breaking: + area: REST API + title: Remove legacy params from range query + details: "The deprecated range query parameters `to`, `from`, `include_lower`, and `include_upper` are no longer supported." + impact: "Users should use `lt`, `lte`, `gt`, and `gte` query parameters instead." + notable: false + essSettingChange: false + - pr: 116996 + summary: Initial work on `ReindexDatastreamIndexAction` + area: Data streams + type: enhancement + issues: [] + - pr: 117095 + summary: Add version prefix to Inference Service API path + area: Inference + type: enhancement + issues: [] + - pr: 117148 + summary: Preserve thread context when waiting for segment generation in RTG + area: CRUD + type: bug + issues: [] + - pr: 117176 + summary: Integrate IBM watsonx to Inference API for re-ranking task + area: Experiences + type: enhancement + issues: [] + - pr: 117199 + summary: Speed up bit compared with floats or bytes script operations + area: Vector Search + type: enhancement + issues: [] + - pr: 117214 + summary: Returning ignored fields in the simulate ingest API + area: Ingest Node + type: enhancement + issues: [] + - pr: 117229 + summary: "In this pr, a 400 error is returned when _source / _seq_no / _feature / _nested_path / _field_names is requested, rather a 5xx" + area: Search + type: bug + issues: [] + - pr: 117230 + summary: Make various alias retrieval APIs wait for cluster to unblock + area: Distributed + type: enhancement + issues: [] + - pr: 117243 + summary: Bump major version for feature migration system indices + area: Infra/Core + type: upgrade + issues: [] + - pr: 117246 + summary: LOOKUP JOIN using field-caps for field mapping + area: ES|QL + type: enhancement + issues: [] + - pr: 117265 + summary: Async search responses have CCS metadata while searches are running + area: ES|QL + type: enhancement + issues: [] + - pr: 117287 + summary: Fixing bug setting index when parsing Google Vertex AI results + area: Machine Learning + type: bug + issues: [] + - pr: 117303 + summary: Remove HTTP content copies + area: Network + type: enhancement + issues: [] + - pr: 117359 + summary: Term query for ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 117451 + summary: ST_EXTENT aggregation + area: ES|QL + type: feature + issues: + - 104659 + - pr: 117519 + summary: Remove `data_frame_transforms` roles + area: Transform + type: breaking + issues: [] + breaking: + area: Transform + title: Remove `data_frame_transforms` roles + details: '`data_frame_transforms_admin` and `data_frame_transforms_user` were deprecated in Elasticsearch 7 and are being removed in Elasticsearch 9. `data_frame_transforms_admin` is now `transform_admin`. `data_frame_transforms_user` is now `transform_user`. Users must call the `_update` API to replace the permissions on the Transform before the Transform can be started.' + impact: "Transforms created with either the `data_frame_transforms_admin` or the `data_frame_transforms_user` role will fail to start. The Transform will remain in a `stopped` state, and its health will be red while displaying permission failures." + notable: false + essSettingChange: false + - pr: 117530 + summary: Expose operation and request counts separately in repository stats + area: Snapshot/Restore + type: enhancement + issues: + - 104443 + - pr: 117555 + summary: Expand type compatibility for match function and operator + area: ES|QL + type: feature + issues: [] + - pr: 117572 + summary: Address and remove any references of RestApiVersion version 7 + area: Search + type: enhancement + issues: [] + - pr: 117581 + summary: Make reserved built-in roles queryable + area: Authorization + type: enhancement + issues: [] + - pr: 117583 + summary: Removing index alias creation for deprecated transforms notification index + area: Machine Learning + type: deprecation + issues: [] + deprecation: + area: Transform + title: Removing index alias creation for deprecated transforms notification index + details: "As part of the migration from 7.x to 8.x, the `.data-frame-notifications-1` index was deprecated and replaced with the `.transform-notifications-000002` index. The index is no longer created by default, all writes are directed to the new index, and any clusters with the deprecated index will have an alias created to ensure that reads are still retrieving data that was written to the index before the migration to 8.x. This change removes the alias from the deprecated index in 9.x. Any clusters with the alias present will retain it, but it will not be created on new clusters." + impact: No known end user impact. + notable: false + essSettingChange: false + - pr: 117589 + summary: Add Inference Unified API for chat completions for OpenAI + area: Machine Learning + type: enhancement + issues: [] + - pr: 117606 + summary: Remove deprecated sort from reindex operation within dataframe analytics procedure + area: Machine Learning + type: enhancement + issues: [] + - pr: 117618 + summary: SearchStatesIt failures reported by CI + area: Search + type: bug + issues: + - 116617 + - 116618 + - pr: 117642 + summary: Adding endpoint creation validation to `ElasticInferenceService` + area: Machine Learning + type: enhancement + issues: [] + - pr: 117643 + summary: Drop null columns in text formats + area: ES|QL + type: bug + issues: + - 116848 + - pr: 117655 + summary: Add nulls support to Categorize + area: ES|QL + type: enhancement + issues: [] + - pr: 117701 + summary: Watcher history index has too many indexed fields - + area: Watcher + type: bug + issues: + - 71479 + - pr: 117731 + summary: Add cluster level reduction + area: ES|QL + type: enhancement + issues: [] + - pr: 117748 + summary: Add IMDSv2 support to `repository-s3` + area: Snapshot/Restore + type: enhancement + issues: + - 105135 + - pr: 117750 + summary: '`CrossClusterIT` `testCancel` failure' + area: Search + type: bug + issues: + - 108061 + - pr: 117778 + summary: "[Connector APIs] Enforce index prefix for managed connectors" + area: Extract&Transform + type: feature + issues: [] + - pr: 117831 + summary: Fix/QueryBuilderBWCIT_muted_test + area: Search + type: bug + issues: [] + - pr: 117839 + summary: Add match support for `semantic_text` fields + area: Search + type: enhancement + issues: [] + - pr: 117858 + summary: Create upgrade mode + area: Transform + type: enhancement + issues: [] + - pr: 117898 + summary: Limit size of query + area: ES|QL + type: bug + issues: [] + - pr: 117917 + summary: Add option to store `sparse_vector` outside `_source` + area: Mapping + type: feature + issues: [] + - pr: 117933 + summary: Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` + area: Infra/Logging + type: bug + issues: + - 83251 + breaking: + area: Logging + title: Deprecation logging value change for "data_stream.dataset" and "event.dataset" + details: |- + This change modifies the "data_stream.dataset" and "event.dataset" value for deprecation logging + to use the value `elasticsearch.deprecation` instead of `deprecation.elasticsearch`. This is now + consistent with other values where the name of the service is the first part of the key. + impact: |- + If you are directly consuming deprecation logs for "data_stream.dataset" and "event.dataset" and filtering on + this value, you will need to update your filters to use `elasticsearch.deprecation` instead of + `deprecation.elasticsearch`. + notable: false + essSettingChange: false + - pr: 117939 + summary: Adding default endpoint for Elastic Rerank + area: Machine Learning + type: enhancement + issues: [] + - pr: 117949 + summary: Move `SlowLogFieldProvider` instantiation to node construction + area: Infra/Logging + type: bug + issues: [] + - pr: 117963 + summary: '`SearchServiceTests.testParseSourceValidation` failure' + area: Search + type: bug + issues: [] + - pr: 117989 + summary: ESQL Add esql hash function + area: ES|QL + type: enhancement + issues: [] + - pr: 117994 + summary: Even better(er) binary quantization + area: Vector Search + type: enhancement + issues: [] + - pr: 118016 + summary: Propagate status codes from shard failures appropriately + area: Search + type: enhancement + issues: + - 118482 + - pr: 118025 + summary: Update sparse text embeddings API route for Inference Service + area: Inference + type: enhancement + issues: [] + - pr: 118027 + summary: Esql compare nanos and millis + area: ES|QL + type: enhancement + issues: + - 116281 + - pr: 118035 + summary: Include hidden indices in `DeprecationInfoAction` + area: Indices APIs + type: bug + issues: + - 118020 + - pr: 118058 + summary: Grant necessary Kibana application privileges to `reporting_user` role + area: Authorization + type: enhancement + issues: [] + - pr: 118064 + summary: Add Highlighter for Semantic Text Fields + area: Highlighting + type: feature + issues: [] + - pr: 118094 + summary: Update ASM 9.7 -> 9.7.1 to support JDK 24 + area: Infra/Core + type: upgrade + issues: [] + - pr: 118102 + summary: "ESQL: Enterprise license enforcement for CCS" + area: ES|QL + type: enhancement + issues: [] + - pr: 118103 + summary: Remove any references to org.elasticsearch.core.RestApiVersion#V_7 + area: Infra/Core + type: breaking + issues: [] + breaking: + area: REST API + title: Remove any references to org.elasticsearch.core.RestApiVersion#V_7 + details: This PR removes all references to V_7 in the Rest API. V7 features marked for deprecation have been removed. + impact: This change is breaking for any external plugins/clients that rely on the V_7 enum or deprecated version 7 functionality + notable: false + essSettingChange: false + - pr: 118104 + summary: Remove old `_knn_search` tech preview API in v9 + area: Vector Search + type: breaking + issues: [] + breaking: + area: REST API + title: Remove old `_knn_search` tech preview API in v9 + details: "The original, tech-preview api for vector search, `_knn_search`, has been removed in v9. For all vector search operations, you should utilize the `_search` endpoint." + impact: The `_knn_search` API is now inaccessible without providing a compatible-with flag for v8. + notable: false + essSettingChange: false + - pr: 118114 + summary: Enable physical plan verification + area: ES|QL + type: enhancement + issues: [] + - pr: 118122 + summary: "ES|QL: Partial result on demand for async queries" + area: ES|QL + type: enhancement + issues: [] + - pr: 118143 + summary: Infrastructure for assuming cluster features in the next major version + area: Infra/Core + type: feature + issues: [] + - pr: 118173 + summary: ES|QL categorize with multiple groupings + area: Machine Learning + type: feature + issues: [] + - pr: 118177 + summary: Fixing bedrock event executor terminated cache issue + area: Machine Learning + type: bug + issues: + - 117916 + - pr: 118188 + summary: Check for early termination in Driver + area: ES|QL + type: enhancement + issues: [] + - pr: 118192 + summary: Remove `client.type` setting + area: Infra/Core + type: breaking + issues: + - 104574 + breaking: + area: Cluster and node setting + title: Remove `client.type` setting + details: The node setting `client.type` has been ignored since the node client was removed in 8.0. The setting is now removed. + impact: Remove the `client.type` setting from `elasticsearch.yml` + notable: false + essSettingChange: false + - pr: 118194 + summary: Retry on `ClusterBlockException` on transform destination index + area: Machine Learning + type: enhancement + issues: [] + - pr: 118266 + summary: Prevent data nodes from sending stack traces to coordinator when `error_trace=false` + area: Search + type: enhancement + issues: [] + - pr: 118267 + summary: Adding get migration reindex status + area: Data streams + type: enhancement + issues: [] + - pr: 118291 + summary: Adding a migration reindex cancel API + area: Data streams + type: enhancement + issues: [] + - pr: 118324 + summary: Allow the data type of `null` in filters + area: ES|QL + type: bug + issues: + - 116351 + - pr: 118353 + summary: Epoch Millis Rounding Down and Not Up 2 + area: Infra/Core + type: bug + issues: [] + - pr: 118366 + summary: |- + Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password + will prevent node from starting + area: Authentication + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: -| Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password will prevent node from starting + details: "-| For LDAP or AD authentication realms, setting a bind DN (via the `xpack.security.authc.realms.ldap.*.bind_dn` or `xpack.security.authc.realms.active_directory.*.bind_dn` realm settings) without a bind password is a misconfiguration that may prevent successful authentication to the node. Nodes will fail to start if a bind DN is specified without a password." + impact: "-| If you have a bind DN configured for an LDAP or AD authentication realm, set a bind password for {ref}/ldap-realm.html#ldap-realm-configuration[LDAP] or {ref}/active-directory-realm.html#ad-realm-configuration[Active Directory]. Configuring a bind DN without a password prevents the misconfigured node from starting." + notable: false + essSettingChange: false + - pr: 118375 + summary: Check for presence of error object when validating streaming responses from integrations in the inference API + area: Machine Learning + type: enhancement + issues: [] + - pr: 118410 + summary: Push down filter passed lookup join + area: ES|QL + type: enhancement + issues: [] + - pr: 118435 + summary: '`_score` should not be a reserved attribute in ES|QL' + area: ES|QL + type: enhancement + issues: + - 118460 + - pr: 118474 + summary: Esql bucket function for date nanos + area: ES|QL + type: enhancement + issues: + - 118031 + - pr: 118484 + summary: Remove date histogram boolean support + area: Aggregations + type: breaking + issues: [] + breaking: + area: Aggregations + title: Remove date histogram boolean support + details: "Elasticsearch no longer allows running Date Histogram aggregations over boolean fields. Instead, use Terms aggregation for boolean fields." + impact: "We expect the impact to be minimal, as this never produced good results, and has been deprecated for years." + notable: false + essSettingChange: false + - pr: 118544 + summary: ESQL - Remove restrictions for disjunctions in full text functions + area: ES|QL + type: enhancement + issues: [] + - pr: 118562 + summary: Update data stream deprecations warnings to new format and filter searchable snapshots from response + area: Data streams + type: enhancement + issues: [] + - pr: 118585 + summary: Add a generic `rescorer` retriever based on the search request's rescore functionality + area: Ranking + type: feature + issues: + - 118327 + - pr: 118602 + summary: Limit memory usage of `fold` + area: ES|QL + type: bug + issues: [] + - pr: 118617 + summary: Add support for `sparse_vector` queries against `semantic_text` fields + area: Search + type: enhancement + issues: [] + - pr: 118619 + summary: Optional named arguments for function in map + area: EQL + type: enhancement + issues: [] + - pr: 118634 + summary: "Add undeclared Azure settings, modify test to exercise them" + area: Snapshot/Restore + type: bug + issues: [] + - pr: 118652 + summary: Add Jina AI API to do inference for Embedding and Rerank models + area: Machine Learning + type: enhancement + issues: [] + - pr: 118669 + summary: "[Connector API] Support soft-deletes of connectors" + area: Extract&Transform + type: feature + issues: [] + - pr: 118671 + summary: Adjust `random_score` default field to `_seq_no` field + area: Search + type: breaking + issues: [] + breaking: + area: Search + title: Adjust `random_score` default field to `_seq_no` field + details: "When providing a 'seed' parameter to a 'random_score' function in the 'function_score' query but NOT providing a 'field', the default 'field' is switched from '_id' to '_seq_no'." + impact: The random scoring and ordering may change when providing a 'seed' and not providing a 'field' to a 'random_score' function. + notable: false + essSettingChange: false + - pr: 118674 + summary: Ignore failures from renormalizing buckets in read-only index + area: Machine Learning + type: enhancement + issues: [] + - pr: 118681 + summary: '`ConnectTransportException` returns retryable BAD_GATEWAY' + area: Network + type: enhancement + issues: + - 118320 + - pr: 118697 + summary: Esql implicit casting for date nanos + area: ES|QL + type: enhancement + issues: + - 118476 + - pr: 118704 + summary: Avoid updating settings version in `MetadataMigrateToDataStreamService` when settings have not changed + area: Data streams + type: bug + issues: [] + - pr: 118774 + summary: Apply default k for knn query eagerly + area: Vector Search + type: bug + issues: [] + - pr: 118802 + summary: ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape + area: ES|QL + type: enhancement + issues: [] + - pr: 118804 + summary: Add new experimental `rank_vectors` mapping for late-interaction second order ranking + area: Vector Search + type: feature + issues: [] + highlight: + notable: true + title: Add new experimental `rank_vectors` mapping for late-interaction second order ranking + body: "Late-interaction models are powerful rerankers. While their size and overall cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali." + pr: 118804 + - pr: 118823 + summary: Fix attribute set equals + area: ES|QL + type: bug + issues: [] + - pr: 118825 + summary: "Remove support for type, fields, `copy_to` and boost in metadata field definition" + area: Mapping + type: breaking + issues: [] + breaking: + area: Mapping + title: "Remove support for type, fields, copy_to and boost in metadata field definition" + details: "The type, fields, copy_to and boost parameters are no longer supported in metadata field definition starting with version 9." + impact: "Users providing type, fields, copy_to or boost as part of metadata field definition should remove them from their mappings." + notable: false + essSettingChange: false + - pr: 118858 + summary: Lookup join on multiple join fields not yet supported + area: ES|QL + type: enhancement + issues: [] + - pr: 118870 + summary: Rewrite TO_UPPER/TO_LOWER comparisons + area: ES|QL + type: enhancement + issues: + - 118304 + - pr: 118871 + summary: "[Elastic Inference Service] Add ElasticInferenceService Unified ChatCompletions Integration" + area: Inference + type: enhancement + issues: [] + - pr: 118890 + summary: Add action to create index from a source index + area: Data streams + type: enhancement + issues: [] + - pr: 118931 + summary: Add a `LicenseAware` interface for licensed Nodes + area: ES|QL + type: enhancement + issues: + - 117405 + - pr: 118938 + summary: Hash functions + area: ES|QL + type: enhancement + issues: [] + - pr: 118941 + summary: Allow archive and searchable snapshots indices in N-2 version + area: Recovery + type: enhancement + issues: [] + - pr: 118959 + summary: Allow kibana_system user to manage .reindexed-v8-internal.alerts indices + area: Authorization + type: enhancement + issues: [] + - pr: 118968 + summary: Configure index sorting through index settings for logsdb + area: Logs + type: enhancement + issues: + - 118686 + - pr: 119001 + summary: Add support for specifying reindexing script for system index migration + area: Infra/Core + type: enhancement + issues: [] + - pr: 119003 + summary: Add a `replicate_for` option to the ILM `searchable_snapshot` action + area: ILM+SLM + type: enhancement + issues: [] + - pr: 119007 + summary: Block-writes cannot be added after read-only + area: Data streams + type: bug + issues: + - 119002 + - pr: 119011 + summary: Add support for knn vector queries on `semantic_text` fields + area: Search + type: enhancement + issues: [] + - pr: 119054 + summary: "[Security Solution] allows `kibana_system` user to manage .reindexed-v8-* Security Solution indices" + area: Authorization + type: enhancement + issues: [] + - pr: 119067 + summary: Metrics for indexing failures due to version conflicts + area: CRUD + type: feature + issues: [] + - pr: 119072 + summary: Turn `_source` meta fieldmapper's mode attribute into a no-op + area: Mapping + type: breaking + issues: + - 118596 + breaking: + area: Mapping + title: Turn `_source` meta fieldmapper's mode attribute into a no-op + details: The `mode` mapping attribute of `_source` metadata field mapper has been turned into a no-op. Instead the `index.mapping.source.mode` index setting should be used to configure source mode. + impact: Configuring the `mode` attribute for the `_source` meta field mapper will have no effect on indices created with Elasticsearch 9.0.0 or later. Note that `_source.mode` configured on indices before upgrading to 9.0.0 or later will remain efficive after upgrading. + notable: false + essSettingChange: false + - pr: 119227 + summary: Remove unfreeze REST endpoint + area: Indices APIs + type: breaking + issues: [] + breaking: + area: REST API + title: Remove unfreeze REST endpoint + details: "The `/{index}/_unfreeze` REST endpoint is no longer supported. This API was deprecated, and the corresponding `/{index}/_freeze` endpoint was removed in 8.0." + impact: "None, since it is not possible to have a frozen index in a version which is readable by Elasticsearch 9.0" + notable: false + essSettingChange: false + - pr: 119250 + summary: Add rest endpoint for `create_from_source_index` + area: Data streams + type: enhancement + issues: [] + - pr: 119265 + summary: Fix `AbstractShapeGeometryFieldMapperTests` + area: ES|QL + type: bug + issues: + - 119201 + - pr: 119291 + summary: Register mustache size limit setting + area: Infra/Scripting + type: bug + issues: [] + - pr: 119296 + summary: Fix writing for LOOKUP status + area: ES|QL + type: bug + issues: + - 119086 + - pr: 119308 + summary: Upgrade to Lucene 10.1.0 + area: Search + type: upgrade + issues: [] + - pr: 119310 + summary: Remove ChunkedToXContentBuilder + area: Network + type: bug + issues: + - 118647 + - pr: 119348 + summary: Auto-migrate `max_page_search_size` + area: Transform + type: enhancement + issues: [] + - pr: 119474 + summary: Add ES|QL cross-cluster query telemetry collection + area: ES|QL + type: enhancement + issues: [] + - pr: 119476 + summary: Fix TopN row size estimate + area: ES|QL + type: bug + issues: + - 106956 + - pr: 119503 + summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. + area: Search + type: bug + issues: + - 117042 + - pr: 119504 + summary: Optimized index sorting for OTel logs + area: Data streams + type: enhancement + issues: [] + - pr: 119536 + summary: Fix ROUND() with unsigned longs throwing in some edge cases + area: ES|QL + type: bug + issues: [] + - pr: 119542 + summary: Wait while index is blocked + area: Transform + type: enhancement + issues: [] + - pr: 119543 + summary: "[Inference API] Fix unique ID message for inference ID matches trained model ID" + area: Machine Learning + type: bug + issues: + - 111312 + - pr: 119546 + summary: Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields + area: Mapping + type: enhancement + issues: [] + - pr: 119564 + summary: Http stream activity tracker and exceptions handling + area: Network + type: enhancement + issues: [] + - pr: 119575 + summary: Fix realtime get of nested fields with synthetic source + area: Mapping + type: bug + issues: + - 119553 + - pr: 119580 + summary: Do not serialize `EsIndex` in plan + area: ES|QL + type: enhancement + issues: [] + - pr: 119621 + summary: Enable node-level reduction by default + area: ES|QL + type: enhancement + issues: [] + - pr: 119679 + summary: Support mTLS for the Elastic Inference Service integration inside the inference API + area: Machine Learning + type: feature + issues: [] + - pr: 119691 + summary: Fix `bbq_hnsw` merge file cleanup on random IO exceptions + area: Vector Search + type: bug + issues: + - 119392 + - pr: 119730 + summary: Enable KQL function as a tech preview + area: ES|QL + type: enhancement + issues: [] + - pr: 119743 + summary: POC mark read-only + area: Engine + type: enhancement + issues: [] + - pr: 119749 + summary: Strengthen encryption for elasticsearch-keystore tool to AES 256 + area: Infra/CLI + type: enhancement + issues: [] + - pr: 119772 + summary: ESQL Support IN operator for Date nanos + area: ES|QL + type: enhancement + issues: + - 118578 + - pr: 119780 + summary: Add index and reindex request settings to speed up reindex + area: Data streams + type: enhancement + issues: [] + - pr: 119792 + summary: Make semantic text part of the text family + area: Search + type: enhancement + issues: [] + - pr: 119798 + summary: "Add a `PostAnalysisAware,` distribute verification" + area: ES|QL + type: enhancement + issues: [] + - pr: 119830 + summary: Run `TransportGetComposableIndexTemplate` on local node + area: Indices APIs + type: enhancement + issues: [] + - pr: 119831 + summary: Run `TransportClusterGetSettingsAction` on local node + area: Infra/Settings + type: enhancement + issues: [] + - pr: 119837 + summary: Run `TransportGetIndexTemplateAction` on local node + area: Indices APIs + type: enhancement + issues: [] + - pr: 119846 + summary: Drop support for brackets from METADATA syntax + area: ES|QL + type: deprecation + issues: + - 115401 + deprecation: + area: ES|QL + title: Drop support for brackets from METADATA syntax + details: Please describe the details of this change for the release notes. You can use asciidoc. + impact: Please describe the impact of this change to users + notable: false + essSettingChange: false + - pr: 119863 + summary: Restrict Connector APIs to manage/monitor_connector privileges + area: Extract&Transform + type: breaking + issues: [] + breaking: + area: REST API + title: Restrict Connector APIs to manage/monitor_connector privileges + details: "Connector APIs now enforce the manage_connector and monitor_connector privileges (introduced in 8.15), replacing the previous reliance on index-level permissions for .elastic-connectors and .elastic-connectors-sync-jobs in API calls." + impact: Connector APIs now require manage_connector and monitor_connector privileges + notable: false + essSettingChange: false + - pr: 119886 + summary: Initial support for unmapped fields + area: ES|QL + type: feature + issues: [] + - pr: 119889 + summary: Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` + area: ES|QL + type: enhancement + issues: [] + - pr: 119893 + summary: Add enterprise license check for Inference API actions + area: Machine Learning + type: enhancement + issues: [] + - pr: 119898 + summary: Resolve/cluster allows querying for cluster info only (no index expression required) + area: CCS + type: enhancement + issues: [] + - pr: 119922 + summary: "[Inference API] fix spell words: covertToString to convertToString" + area: Machine Learning + type: enhancement + issues: [] + - pr: 119926 + summary: Deprecated tracing.apm.* settings got removed. + area: Infra/Metrics + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Deprecated tracing.apm.* settings got removed. + details: "Deprecated `tracing.apm.*` settings got removed, use respective `telemetry.*` / `telemetry.tracing.*` settings instead." + impact: 9.x nodes will refuse to start if any such setting (including secret settings) is still present. + notable: false + essSettingChange: false + - pr: 119968 + summary: Reset relocation/allocation failure counter on node join/shutdown + area: Allocation + type: enhancement + issues: [] + - pr: 119995 + summary: "apm-data: Use representative count as event.success_count if available" + area: Ingest Node + type: bug + issues: [] + - pr: 120020 + summary: Resume Driver on cancelled or early finished + area: ES|QL type: enhancement issues: [] - - pr: 117642 - summary: Adding endpoint creation validation to `ElasticInferenceService` + - pr: 120038 + summary: Run template simulation actions on local node + area: Ingest Node + type: enhancement + issues: [] + - pr: 120042 + summary: Match dot prefix of migrated DS backing index with the source index + area: Data streams + type: bug + issues: [] + - pr: 120055 + summary: Optimize loading mappings when determining synthetic source usage and whether host.name can be sorted on. + area: Logs + type: enhancement + issues: [] + - pr: 120064 + summary: Change the auditor to write via an alias area: Machine Learning + type: upgrade + issues: [] + - pr: 120084 + summary: Improve how reindex data stream index action handles api blocks + area: Data streams type: enhancement issues: [] - - pr: 119503 - summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. - area: Search + - pr: 120087 + summary: Include `clusterApplyListener` in long cluster apply warnings + area: Cluster Coordination + type: enhancement + issues: [] + - pr: 120108 + summary: Remove the ability to read frozen indices + area: Indices APIs + type: breaking + issues: [] + breaking: + area: Index setting + title: Remove the ability to read frozen indices + details: The ability to read frozen indices has been removed. (Frozen indices are no longer useful due to improvements in heap memory usage. The ability to freeze indices was removed in 8.0.) + impact: Users must unfreeze any frozen indices before upgrading. + notable: false + essSettingChange: false + - pr: 120128 + summary: Add Multi-Field Support for Semantic Text Fields + area: Relevance + type: feature + issues: [] + - pr: 120142 + summary: Limit `ByteSizeUnit` to 2 decimals + area: Infra/Core + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Limit `ByteSizeUnit` to 2 decimals + details: "In the past, byte values like `1.25 mb` were allowed but deprecated. Now, values with up to two decimal places are allowed, unless the unit is bytes, in which case no decimals are allowed. Values with too many decimal places result in an error." + impact: "Values with more than two decimal places, like `0.123 mb` will be rejected as an error, where in the past, they'd be accepted with a deprecation warning." + notable: false + essSettingChange: false + - pr: 120143 + summary: Esql - support date nanos in date format function + area: ES|QL + type: enhancement + issues: + - 109994 + - pr: 120163 + summary: Filter deprecated settings when making dest index + area: Data streams + type: enhancement + issues: [] + - pr: 120192 + summary: Extend `TranslationAware` to all pushable expressions + area: ES|QL + type: enhancement + issues: [] + - pr: 120193 + summary: "Do not capture `ClusterChangedEvent` in `IndicesStore` call to #onClusterStateShardsClosed" + area: Store type: bug + issues: [] + - pr: 120198 + summary: Bump `TrialLicenseVersion` to allow starting new trial on 9.0 + area: License + type: enhancement + issues: [] + - pr: 120200 + summary: "[Connector API] Support hard deletes with new URL param in delete endpoint" + area: Extract&Transform + type: feature + issues: [] + - pr: 120207 + summary: Make `requests_per_second` configurable to throttle reindexing + area: Data streams + type: enhancement + issues: [] + - pr: 120222 + summary: Adding linear retriever to support weighted sums of sub-retrievers + area: Search + type: enhancement + issues: [] + - pr: 120231 + summary: Add sanity check to `ReindexDatastreamIndexAction` + area: Data streams + type: enhancement + issues: [] + - pr: 120244 + summary: Ignore closed indices for reindex + area: Data streams + type: enhancement + issues: [] + - pr: 120250 + summary: "Retry internally when CAS upload is throttled [GCS]" + area: Snapshot/Restore + type: enhancement issues: - - 117042 - - pr: 119546 - summary: Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields + - 116546 + - pr: 120267 + summary: Set allow_partial_search_results=true by default + area: EQL + type: breaking + issues: [] + breaking: + area: REST API + title: Set allow_partial_search_results=true by default + details: "Before this change, in case of shard failures, EQL queries always returned an error. With this change, they will keep running and will return partial results." + impact: "EQL queries that would previously fail due to shard failures, will now succeed and return partial results. The previous defaults can be restored by setting `xpack.eql.default_allow_partial_results` cluster setting to `false` or setting with `allow_partial_search_results` to `false` in the query request." + notable: false + essSettingChange: false + - pr: 120271 + summary: Optimize indexing points with index and doc values set to true + area: Geo + type: enhancement + issues: [] + - pr: 120291 + summary: ESQL - Allow full text functions disjunctions for non-full text functions + area: ES|QL + type: feature + issues: [] + - pr: 120334 + summary: Introduce `IndexSettingDeprecatedInV8AndRemovedInV9` Setting property + area: Infra/Settings + type: enhancement + issues: [] + - pr: 120340 + summary: Add support for `extended_stats` + area: Transform + type: enhancement + issues: [] + - pr: 120343 + summary: Support some stats on aggregate_metric_double + area: ES|QL + type: enhancement + issues: + - 110649 + - pr: 120354 + summary: Move scoring in ES|QL out of snapshot + area: ES|QL + type: enhancement + issues: [] + - pr: 120355 + summary: Ensure cluster string could be quoted + area: ES|QL + type: enhancement + issues: [] + - pr: 120360 + summary: ESQL - Add Match function options + area: ES|QL + type: feature + issues: [] + - pr: 120370 + summary: "Merge field mappers when updating mappings with [subobjects:false]" area: Mapping + type: bug + issues: + - 120216 + - pr: 120392 + summary: Test/107515 restore template with match only text mapper it fail + area: Search + type: bug + issues: + - 107515 + - pr: 120400 + summary: "[Inference API] Add node-local rate limiting for the inference API" + area: Machine Learning + type: feature + issues: [] + - pr: 120405 + summary: Automatically rollover legacy ml indices + area: Machine Learning + type: upgrade + issues: [] + - pr: 120445 + summary: Run `GetPipelineTransportAction` on local node + area: Ingest Node type: enhancement issues: [] - - pr: 119995 - summary: "apm-data: Use representative count as event.success_count if available" + - pr: 120458 + summary: Do not recommend increasing `max_shards_per_node` + area: Health + type: bug + issues: [] + - pr: 120487 + summary: Fix cat_component_templates documentation + area: CAT APIs + type: bug + issues: [] + - pr: 120494 + summary: Update grammar to rely on `indexPattern` instead of identifier in join target + area: ES|QL + type: enhancement + issues: [] + - pr: 120505 + summary: introduce new categories for deprecated resources in deprecation API + area: Indices APIs + type: enhancement + issues: [] + - pr: 120538 + summary: "Revert unwanted ES|QL lexer changes from PR #120354" + area: ES|QL + type: bug + issues: [] + - pr: 120542 + summary: "Feat: add a user-configurable timeout parameter to the `_resolve/cluster` API" + area: Search + type: enhancement + issues: [] + - pr: 120546 + summary: Restrict agent entitlements to the system classloader unnamed module + area: Infra/Plugins + type: bug + issues: [] + - pr: 120547 + summary: Consistent mapping for OTel log and event bodies + area: Data streams + type: enhancement + issues: [] + - pr: 120548 + summary: Add `remove_index_block` arg to `_create_from` api + area: Indices APIs + type: enhancement + issues: [] + - pr: 120551 + summary: Set default reranker for text similarity reranker to Elastic reranker + area: Ranking + type: enhancement + issues: [] + - pr: 120573 + summary: Optimize `IngestDocument` `FieldPath` allocation area: Ingest Node + type: enhancement + issues: [] + - pr: 120591 + summary: Increase field limit for OTel metrics to 10 000 + area: TSDB + type: enhancement + issues: [] + - pr: 120617 + summary: Fix queries with document level security on lookup indexes + area: ES|QL + type: bug + issues: + - 120509 + - pr: 120629 + summary: Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As A Warning + area: Data streams + type: enhancement + issues: [] + - pr: 120642 + summary: Defer unpromotable shard refreshes until index refresh blocks are cleared + area: Engine + type: enhancement + issues: [] + - pr: 120643 + summary: Remove index blocks by default in `create_from` + area: Indices APIs + type: enhancement + issues: [] + - pr: 120645 + summary: Esql Support date nanos on date diff function + area: ES|QL + type: enhancement + issues: + - 109999 + - pr: 120662 + summary: Fix broken yaml test `30_create_from` + area: Indices APIs type: bug issues: [] + - pr: 120722 + summary: Migrate stream to core error parsing + area: Machine Learning + type: enhancement + issues: [] + - pr: 120725 + summary: |- + A new query parameter `?include_source_on_error` was added for create / index, update and bulk REST APIs to control + if to include the document source in the error response in case of parsing errors. The default value is `true`. + area: Infra/REST API + type: enhancement + issues: [] + - pr: 120727 + summary: Esql - Support date nanos in date extract function + area: ES|QL + type: enhancement + issues: + - 110000 + - pr: 120748 + summary: Removing support for types field in watcher search + area: Watcher + type: breaking + issues: [] + breaking: + area: REST API + title: Removing support for types field in watcher search + details: "Previously, setting the `input.search.request.types` field in the payload when creating a watcher to an empty array was allowed, although it resulted in a deprecation warning and had no effect (and any value other than an empty array would result in an error). Now, support for this field is entirely removed, and the empty array will also result in an error." + impact: Users should stop setting this field (which did not have any effect anyway). + notable: false + essSettingChange: false - pr: 120751 summary: Adding support for binary embedding type to Cohere service embedding type area: Machine Learning type: enhancement issues: [] + - pr: 120752 + summary: Refresh source index before reindexing data stream index + area: Data streams + type: bug + issues: + - 120314 + - pr: 120753 + summary: Optimize `IngestDocMetadata` `isAvailable` + area: Ingest Node + type: enhancement + issues: [] - pr: 120774 summary: Retry ES|QL node requests on shard level failures area: ES|QL type: enhancement issues: [] + - pr: 120799 + summary: Remove duplicate code in ESIntegTestCase + area: Search + type: bug + issues: [] + - pr: 120806 + summary: Deprecate certificate based remote cluster security model + area: Security + type: deprecation + issues: [] + deprecation: + area: Authorization + title: Deprecate certificate based remote cluster security model + details: "-| <> is deprecated and will be removed in a future major version. Users are encouraged to <>. The <> is preferred way to configure remote clusters, as it allows to follow security best practices when setting up remote cluster connections and defining fine-grained access control." + impact: "-| If you have configured remote clusters with certificate-based security model, you should <>. Configuring a remote cluster using <>, generates a warning in the deprecation logs." + notable: false + essSettingChange: false + - pr: 120807 + summary: Remove INDEX_REFRESH_BLOCK after index becomes searchable + area: CRUD + type: enhancement + issues: [] + - pr: 120813 + summary: Change Semantic Text To Act Like A Normal Text Field + area: Search + type: breaking + issues: [] + breaking: + area: Search + title: Change Semantic Text To Act Like A Normal Text Field + details: "The previous semantic_text format used a complex subfield structure in _source to store the embeddings. This complicated interactions/integrations with semantic_text fields and _source in general. This new semantic_text format treats it as a normal text field, where the field's value in _source is the value assigned by the user." + impact: "Users who parsed the subfield structure of the previous semantic_text format in _source will need to update their parsing logic. The new format does not directly expose the chunks and embeddings generated from the input text. The new format will be applied to all new indices, any existing indices will continue to use the previous format." + notable: true + essSettingChange: false + - pr: 120821 + summary: "[Deprecation] Add `transform_ids` to outdated index" + area: Transform + type: enhancement + issues: [] + - pr: 120824 + summary: Optimize some per-document hot paths in the geoip processor + area: Ingest Node + type: enhancement + issues: [] + - pr: 120833 + summary: Optimize `IngestCtxMap` construction + area: Ingest Node + type: enhancement + issues: [] + - pr: 120842 + summary: Remove Elastic Inference Service feature flag and deprecated setting + area: Inference + type: enhancement + issues: [] + - pr: 120852 + summary: Correct line and column numbers of missing named parameters + area: ES|QL + type: bug + issues: [] - pr: 120869 summary: Threadpool merge scheduler area: Engine type: feature issues: [] + - pr: 120883 + summary: Add `?master_timeout` to `POST /_ilm/migrate_to_data_tiers` + area: Indices APIs + type: bug + issues: [] + - pr: 120913 + summary: Automatically rollover legacy .ml-anomalies indices + area: Machine Learning + type: upgrade + issues: [] + - pr: 120937 + summary: Use the system index descriptor in the snapshot blob cache cleanup task + area: Snapshot/Restore + type: bug + issues: + - 120518 - pr: 120957 summary: Introduce `AllocationBalancingRoundSummaryService` area: Allocation type: enhancement issues: [] - - pr: 120998 - summary: ES|QL `change_point` processing command - area: Machine Learning + - pr: 120959 + summary: Remove unnecessary entitlement + area: Infra/Plugins + type: bug + issues: [] + - pr: 120974 + summary: Tweak `copy_to` handling in synthetic `_source` to account for nested objects + area: Mapping + type: bug + issues: + - 120831 + - pr: 120997 + summary: Allow `SSHA-256` for API key credential hash + area: Authentication + type: enhancement + issues: [] + - pr: 120998 + summary: ES|QL `change_point` processing command + area: Machine Learning + type: feature + issues: [] + - pr: 121048 + summary: Updating Inference Update API documentation to have the correct PUT method + area: Machine Learning + type: bug + issues: [] + - pr: 121049 + summary: Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. + area: Logs + type: breaking + issues: + - 106489 + breaking: + area: Logs + title: Conditionally enable logsdb by default + details: |- + Logsdb will be enabled by default for data streams matching with logs-*-* pattern. + If upgrading from 8.x to 9.x and data streams matching with log-*-* do exist, + then Logsdb will not be enabled by default. + impact: |- + Logsdb reduce storage footprint in Elasticsearch for logs, but there are side effects + to be taken into account that are described in the Logsdb docs: + https://www.elastic.co/guide/en/elasticsearch/reference/current/logs-data-stream.html#upgrade-to-logsdb-notes + notable: true + essSettingChange: false + - pr: 121074 + summary: Implement a `MetricsAware` interface + area: ES|QL + type: enhancement + issues: [] + - pr: 121105 + summary: Mark bbq indices as GA and add rolling upgrade integration tests + area: Vector Search type: feature issues: [] - pr: 121106 @@ -63,6 +1954,51 @@ changelogs: area: Machine Learning type: enhancement issues: [] + - pr: 121109 + summary: Fix propagation of dynamic mapping parameter when applying `copy_to` + area: Mapping + type: bug + issues: + - 113049 + - pr: 121124 + summary: Run `TransportGetEnrichPolicyAction` on local node + area: Ingest Node + type: enhancement + issues: [] + - pr: 121156 + summary: Remove redundant sorts from execution plan + area: ES|QL + type: bug + issues: [] + - pr: 121193 + summary: Enable LOOKUP JOIN in non-snapshot builds + area: ES|QL + type: enhancement + issues: + - 121185 + highlight: + notable: true + title: Enable LOOKUP JOIN in non-snapshot builds + body: |- + This effectively releases LOOKUP JOIN into tech preview. Docs will + follow in a separate PR. + + - Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds. + - Remove the grammar for the unsupported `| JOIN ...` command (without `LOOKUP` as first keyword). The way the lexer modes work, otherwise we'd also have to enable `| JOIN ...` syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages. + - Remove grammar for `LOOKUP JOIN index AS ...` because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries. + + Also fix https://github.com/elastic/elasticsearch/issues/121185 + pr: 121193 + - pr: 121196 + summary: Fix geoip databases index access after system feature migration + area: Ingest Node + type: bug + issues: [] + - pr: 121207 + summary: "[Inference API] Put back legacy EIS URL setting" + area: Inference + type: bug + issues: [] - pr: 121240 summary: Implement runtime skip_unavailable=true area: ES|QL @@ -84,6 +2020,11 @@ changelogs: type: bug issues: - 82432 + - pr: 121325 + summary: '`ReindexDataStreamIndex` bug in assertion caused by reference equality' + area: Data streams + type: bug + issues: [] - pr: 121327 summary: Reduce Data Loss in System Indices Migration area: Infra/Core @@ -94,26 +2035,116 @@ changelogs: area: ILM+SLM type: enhancement issues: [] + - pr: 121392 + summary: Include data streams when converting an existing resource to a system resource + area: Infra/Core + type: bug + issues: [] + - pr: 121396 + summary: Change format for Unified Chat + area: Machine Learning + type: bug + issues: [] - pr: 121548 summary: Adding support for specifying embedding type to Jina AI service settings area: Machine Learning type: enhancement issues: [] + - pr: 121552 + summary: Fix a bug in TOP + area: ES|QL + type: bug + issues: [] + - pr: 121556 + summary: Enable New Semantic Text Format Only On Newly Created Indices + area: Mapping + type: bug + issues: [] + - pr: 121568 + summary: Analyze API to return 400 for wrong custom analyzer + area: Analysis + type: bug + issues: + - 121443 + - pr: 121667 + summary: Add deprecation warning for flush API + area: Machine Learning + type: deprecation + issues: + - 121506 + deprecation: + area: REST API + title: Add deprecation warning for flush API + details: "The anomaly detection job flush API is deprecated since it is only required for the post data API, which was deprecated since 7.11.0." + impact: "This should have a minimal impact on users as the flush API is only required for the post data API, which was deprecated since 7.11.0." + notable: false + essSettingChange: false + - pr: 121720 + summary: Skip fetching _inference_fields field in legacy semantic_text format + area: Search + type: bug + issues: [] + - pr: 121727 + summary: Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` + area: Downsampling + type: bug + issues: + - 119696 + - 96076 + - pr: 121731 + summary: Remove TLSv1.1 from default protocols + area: TLS + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Remove TLSv1.1 from default protocols + details: "TLSv1.1 is no longer enabled by default. Prior to version 9.0, Elasticsearch would attempt to enable TLSv1.1 if the JDK supported it. In most cases, including all cases where Elasticsearch 8 was running with the bundled JDK, the JDK would not support TLSv1.1, so that protocol would not be available in Elasticsearch. However, if Elasticsearch was running on an old JDK or a JDK that have been reconfigured to support TLSv1.1, then the protocol would automatically be available within Elasticsearch. As of Elasticsearch 9.0, this is no longer true. If you wish to enable TLSv1.1 then you must enable it within the JDK and also enable it within Elasticsearch by using the `ssl.supported_protocols` setting." + impact: "Most users will not be impacted. If your Elastisearch 8 cluster was using a custom JDK and you relied on TLSv1.1, then you will need to explicitly enable TLSv1.1 within Elasticsearch (as well as enabling it within your JDK)" + notable: false + essSettingChange: false - pr: 121784 summary: Optionally allow text similarity reranking to fail area: Search type: enhancement issues: [] + - pr: 121787 + summary: Added optional parameters to QSTR ES|QL function + area: Search + type: feature + issues: + - 120933 + - pr: 121793 + summary: ES|QL - Add scoring for full text functions disjunctions + area: ES|QL + type: enhancement + issues: [] - pr: 121805 summary: Support subset of metrics in aggregate metric double area: ES|QL type: enhancement issues: [] + - pr: 121821 + summary: Fix get all inference endponts not returning multiple endpoints sharing model deployment + area: Machine Learning + type: bug + issues: [] - pr: 121827 summary: Updates to allow using Cohere binary embedding response in semantic search queries area: Machine Learning type: bug issues: [] + - pr: 121843 + summary: Fix async stop sometimes not properly collecting result + area: ES|QL + type: bug + issues: + - 121249 + - pr: 121850 + summary: Take named parameters for identifier and pattern out of snapshot + area: ES|QL + type: enhancement + issues: [] - pr: 121885 summary: Introduce batched query execution and data-node side reduce area: Search @@ -135,6 +2166,11 @@ changelogs: area: ES|QL type: feature issues: [] + - pr: 121971 + summary: Do not fetch reserved roles from native store when Get Role API is called + area: Authorization + type: enhancement + issues: [] - pr: 122062 summary: Upgrade `discovery-ec2` to AWS SDK v2 area: Discovery-Plugins @@ -181,6 +2217,12 @@ changelogs: area: Machine Learning type: enhancement issues: [] + - pr: 122074 + summary: "If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias" + area: Transform + type: bug + issues: + - 121913 - pr: 122134 summary: Adding integration for VoyageAI embeddings and rerank models area: Machine Learning @@ -197,22 +2239,53 @@ changelogs: type: enhancement issues: - 120874 + - pr: 122246 + summary: Ensure removal of index blocks does not leave key with null value + area: Data streams + type: bug + issues: [] - pr: 122250 summary: "ESQL: Align `RENAME` behavior with `EVAL` for sequential processing" area: ES|QL type: enhancement issues: - 121739 + - pr: 122257 + summary: Revive inlinestats + area: ES|QL + type: bug + issues: [] - pr: 122272 summary: "[Inference API] Rename `model_id` prop to model in EIS sparse inference request body" area: Inference type: enhancement issues: [] + - pr: 122278 + summary: Fix serialising the inference update request + area: Machine Learning + type: bug + issues: [] - pr: 122280 summary: Use `FallbackSyntheticSourceBlockLoader` for number fields area: Mapping type: enhancement issues: [] + - pr: 122293 + summary: Add enterprise license check to inference action for semantic text fields + area: Machine Learning + type: bug + issues: [] + - pr: 122326 + summary: System Index Migration Failure Results in a Non-Recoverable State + area: Infra/Core + type: bug + issues: [] + - pr: 122357 + summary: Handle search timeout in `SuggestPhase` + area: Search + type: bug + issues: + - 122186 - pr: 122365 summary: Fix handling of auto expand replicas for stateless indices area: Search @@ -229,6 +2302,22 @@ changelogs: area: Health type: enhancement issues: [] + - pr: 122409 + summary: Allow setting the `type` in the reroute processor + area: Ingest Node + type: enhancement + issues: + - 121553 + - pr: 122427 + summary: Improve size limiting string message + area: Infra/Core + type: enhancement + issues: [] + - pr: 122431 + summary: Upgrade AWS SDK to v1.12.746 + area: Snapshot/Restore + type: upgrade + issues: [] - pr: 122458 summary: '`DesiredBalanceReconciler` always returns `AllocationStats`' area: Allocation @@ -244,12 +2333,29 @@ changelogs: area: Data streams type: enhancement issues: [] + - pr: 122575 + summary: Return an empty suggestion when suggest phase times out + area: Suggesters + type: bug + issues: + - 122548 - pr: 122586 summary: "ESQL: Fix inconsistent results in using scaled_float field" area: ES|QL type: bug issues: - 122547 + - pr: 122601 + summary: Implicit numeric casting for CASE/GREATEST/LEAST + area: ES|QL + type: bug + issues: + - 121890 + - pr: 122606 + summary: Avoid serializing empty `_source` fields in mappings + area: Mapping + type: bug + issues: [] - pr: 122615 summary: Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. area: Mapping @@ -267,6 +2373,12 @@ changelogs: type: enhancement issues: - 117856 + - pr: 122653 + summary: Knn vector rescoring to sort score docs + area: Vector Search + type: bug + issues: + - 119711 - pr: 122660 summary: Render `aggregate_metric_double` area: ES|QL @@ -277,6 +2389,17 @@ changelogs: area: ES|QL type: enhancement issues: [] + - pr: 122731 + summary: Fork post-snapshot-delete cleanup off master thread + area: Snapshot/Restore + type: bug + issues: [] + - pr: 122762 + summary: "ESQL: Remove estimated row size assertion" + area: ES|QL + type: bug + issues: + - 121535 - pr: 122821 summary: Fix functions emitting warnings with no source area: ES|QL @@ -308,6 +2431,12 @@ changelogs: area: ILM+SLM type: enhancement issues: [] + - pr: 122886 + summary: Add support to VALUES aggregation for spatial types + area: ES|QL + type: bug + issues: + - 122413 - pr: 122890 summary: Introduce `allow_partial_results` setting in ES|QL area: ES|QL @@ -318,6 +2447,11 @@ changelogs: area: ES|QL type: enhancement issues: [] + - pr: 122905 + summary: Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover + area: Data streams + type: bug + issues: [] - pr: 122921 summary: Run `TransportGetMappingsAction` on local node area: Indices APIs @@ -328,6 +2462,28 @@ changelogs: area: Stats type: enhancement issues: [] + - pr: 122938 + summary: Fix geoip databases index access after system feature migration (again) + area: Ingest Node + type: bug + issues: [] + - pr: 122951 + summary: Updates the deprecation info API to not warn about system indices and data streams + area: Indices APIs + type: bug + issues: [] + - pr: 122960 + summary: Deprecate Behavioral Analytics CRUD apis + area: Search + type: deprecation + issues: [] + deprecation: + area: Search + title: Deprecate Behavioral Analytics CRUD apis + details: "Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. The APIs will still work for now, but will emit warning headers that the API has been deprecated." + impact: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. + notable: false + essSettingChange: false - pr: 122991 summary: "GCS blob store: add `OperationPurpose/Operation` stats counters" area: Snapshot/Restore @@ -338,6 +2494,17 @@ changelogs: area: Mapping type: enhancement issues: [] + - pr: 123010 + summary: Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` + area: Engine + type: bug + issues: + - 122974 + - pr: 123076 + summary: Retry on streaming errors + area: Machine Learning + type: bug + issues: [] - pr: 123085 summary: Remove duplicated nested commands area: ES|QL @@ -348,22 +2515,60 @@ changelogs: area: Machine Learning type: feature issues: [] + - pr: 123155 + summary: Add `ElasticInferenceServiceCompletionServiceSettings` + area: Machine Learning + type: bug + issues: [] - pr: 123187 summary: Add bit vector support to semantic text area: Vector Search type: enhancement issues: [] + - pr: 123272 + summary: Set Connect Timeout to 5s + area: Machine Learning + type: bug + issues: [] + - pr: 123290 + summary: Fix Driver status iterations and `cpuTime` + area: ES|QL + type: enhancement + issues: + - 122967 + - pr: 123296 + summary: Avoid over collecting in Limit or Lucene Operator + area: ES|QL + type: bug + issues: [] + - pr: 123381 + summary: Push down `StartsWith` and `EndsWith` functions to Lucene + area: ES|QL + type: enhancement + issues: + - 123067 - pr: 123396 summary: Add initial grammar and planning for RRF (snapshot) area: ES|QL type: feature issues: [] + - pr: 123427 + summary: Reduce iteration complexity for plan traversal + area: ES|QL + type: bug + issues: [] - pr: 123460 summary: "ES|QL: Support `::date` in inline cast" area: ES|QL type: enhancement issues: - 116746 + - pr: 123492 + summary: Fix function registry concurrency issues on constructor + area: ES|QL + type: bug + issues: + - 123430 - pr: 123543 summary: Enhance memory accounting for document expansion and introduce max document size limit area: CRUD @@ -379,6 +2584,23 @@ changelogs: area: Infra/Core type: enhancement issues: [] + - pr: 123589 + summary: Revive some more of inlinestats functionality + area: ES|QL + type: bug + issues: [] + - pr: 123600 + summary: Drop `TLS_RSA` cipher support for JDK 24 + area: TLS + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Drop `TLS_RSA` cipher support for JDK 24 + details: "This change removes `TLS_RSA` ciphers from the list of default supported ciphers, for Elasticsearch deployments running on JDK 24." + impact: "The dropped ciphers are `TLS_RSA_WITH_AES_256_GCM_SHA384`, `TLS_RSA_WITH_AES_128_GCM_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA`, and `TLS_RSA_WITH_AES_128_CBC_SHA`. TLS connections to Elasticsearch using these ciphers will no longer work. Please configure your clients to use one of supported cipher suites." + notable: false + essSettingChange: false - pr: 123630 summary: Limit number of suppressed S3 deletion errors area: Snapshot/Restore @@ -390,6 +2612,21 @@ changelogs: area: ILM+SLM type: enhancement issues: [] + - pr: 123728 + summary: Allow skip shards with `_tier` and `_index` in ES|QL + area: ES|QL + type: enhancement + issues: [] + - pr: 123743 + summary: Adjust exception thrown when unable to load hunspell dict + area: Analysis + type: bug + issues: [] + - pr: 123761 + summary: Have create index return a bad request on poor formatting + area: Infra/Core + type: bug + issues: [] - pr: 123763 summary: Skip semantic_text embedding generation when no content is provided. area: Relevance @@ -400,6 +2637,12 @@ changelogs: area: Snapshot/Restore type: enhancement issues: [] + - pr: 124001 + summary: Use a must boolean statement when pushing down to Lucene when scoring is also needed + area: ES|QL + type: bug + issues: + - 123967 - pr: 124025 summary: "[Inference API] Propagate product use case http header to EIS" area: Machine Learning @@ -421,28 +2664,73 @@ changelogs: area: ES|QL type: enhancement issues: [] + - pr: 124149 + summary: Retry ILM async action after reindexing data stream + area: Data streams + type: enhancement + issues: [] - pr: 124177 summary: "Improve error message for ( and [" area: ES|QL type: bug issues: - 124145 + - pr: 124225 + summary: "[Inference API] Fix output stream ordering in `InferenceActionProxy`" + area: Machine Learning + type: bug + issues: [] - pr: 124313 summary: Optimize memory usage in `ShardBulkInferenceActionFilter` area: Search type: enhancement issues: [] + - pr: 124335 + summary: Change the order of the optimization rules + area: ES|QL + type: bug + issues: [] + - pr: 124363 + summary: Set cause on create index request in create from action + area: Data streams + type: enhancement + issues: [] - pr: 124394 summary: Avoid `NamedWritable` in block serialization area: ES|QL type: enhancement issues: [] + - pr: 124424 + summary: Lazy collection copying during node transform + area: ES|QL + type: bug + issues: [] - pr: 124446 summary: "ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate`" area: ES|QL type: bug issues: - 124311 + - pr: 124451 + summary: Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. + area: Downsampling + type: bug + issues: [] + - pr: 124477 + summary: Improve downsample performance by buffering docids and do bulk processing + area: Downsampling + type: enhancement + issues: [] + - pr: 124527 + summary: Avoid potentially throwing calls to Task#getDescription in model download + area: Machine Learning + type: bug + issues: [] + - pr: 124540 + summary: "ES|QL: Fix scoring for full text functions" + area: ES|QL + type: bug + issues: [] - pr: 124564 summary: Restore `TextSimilarityRankBuilder` XContent output area: Ranking @@ -468,17 +2756,39 @@ changelogs: area: ES|QL type: enhancement issues: [] + - pr: 124604 + summary: Fix geoip databases index access after system feature migration (take 3) + area: Ingest Node + type: bug + issues: [] - pr: 124610 summary: Remove page alignment in exchange sink area: ES|QL type: enhancement issues: [] + - pr: 124611 + summary: Reuse child `outputSet` inside the plan where possible + area: ES|QL + type: enhancement + issues: [] - pr: 124638 summary: Provide model size statistics as soon as an anomaly detection job is opened area: Machine Learning type: bug issues: - 121168 + - pr: 124651 + summary: Fix system data streams to be restorable from a snapshot + area: Infra/Core + type: bug + issues: + - 89261 + - pr: 124662 + summary: Let MLTQuery throw IAE when no analyzer is set + area: Search + type: bug + issues: + - 124562 - pr: 124669 summary: Release semantic_text as a GA feature area: Mapping @@ -511,11 +2821,22 @@ changelogs: area: Machine Learning type: upgrade issues: [] + - pr: 124739 + summary: Improve rolling up metrics + area: Downsampling + type: enhancement + issues: [] - pr: 124782 summary: "Aggs: Let terms run in global ords mode no match" area: Aggregations type: bug issues: [] + - pr: 124784 + summary: Merge template mappings properly during validation + area: Mapping + type: bug + issues: + - 123372 - pr: 124823 summary: Report failures on partial results area: ES|QL @@ -526,6 +2847,27 @@ changelogs: area: Distributed type: bug issues: [] + - pr: 124843 + summary: Ignore _JAVA_OPTIONS + area: Infra/CLI + type: enhancement + issues: [] + - pr: 124865 + summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." + area: Infra/Core + type: upgrade + issues: [] + - pr: 124873 + summary: Restore V8 REST compatibility around highlight `force_source` parameter + area: Highlighting + type: bug + issues: [] + - pr: 124884 + summary: System data streams are not being upgraded in the feature migration API + area: Infra/Core + type: bug + issues: + - 122949 - pr: 124898 summary: Add cache support in `TransportGetAllocationStatsAction` area: Allocation @@ -557,6 +2899,16 @@ changelogs: area: ES|QL type: enhancement issues: [] + - pr: 124931 + summary: This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. + area: Snapshot/Restore + type: bug + issues: [] + - pr: 124936 + summary: Indicate when errors represent timeouts + area: Infra/REST API + type: enhancement + issues: [] - pr: 125023 summary: Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` area: Machine Learning @@ -568,11 +2920,21 @@ changelogs: type: enhancement issues: - 124181 + - pr: 125103 + summary: Fix LTR query feature with phrases (and two-phase) queries + area: Ranking + type: bug + issues: [] - pr: 125117 summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." area: Infra/Core type: upgrade issues: [] + - pr: 125159 + summary: Update bundled JDK to Java 24 + area: Packaging + type: upgrade + issues: [] - pr: 125171 summary: Reindex data stream indices on different nodes area: Data streams @@ -725,6 +3087,11 @@ changelogs: area: Vector Search type: bug issues: [] + - pr: 125732 + summary: Log stack traces on data nodes before they are cleared for transport + area: Search + type: bug + issues: [] - pr: 125764 summary: Fix `ReplaceMissingFieldsWithNull` area: ES|QL @@ -753,3 +3120,8 @@ changelogs: area: Indices APIs type: bug issues: [] + - pr: 126191 + summary: Fix NPE for missing Content Type header in OIDC Authenticator + area: Authentication + type: bug + issues: [] diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index 4ad739e6749bb..d7d66fdd0b17d 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -18,7 +18,23 @@ To give you insight into what deprecated features you’re using, {{es}}: ## 9.0.0 [elasticsearch-900-deprecations] -No deprecations in this version. +ES|QL: +* Drop support for brackets from METADATA syntax [#119846](https://github.com/elastic/elasticsearch/pull/119846) (issue: {es-issue}115401[#115401]) + +Ingest Node: +* Fix `_type` deprecation on simulate pipeline API [#116259](https://github.com/elastic/elasticsearch/pull/116259) + +Machine Learning: +* Removing index alias creation for deprecated transforms notification index [#117583](https://github.com/elastic/elasticsearch/pull/117583) +* [Inference API] Deprecate elser service [#113216](https://github.com/elastic/elasticsearch/pull/113216) + +Rollup: +* Emit deprecation warning when executing one of the rollup APIs [#113131](https://github.com/elastic/elasticsearch/pull/113131) + +Security: +* Deprecate certificate based remote cluster security model [#120806](https://github.com/elastic/elasticsearch/pull/120806) + + ## 9.1.0 [elasticsearch-910-deprecations] diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index b2372f5d9da68..75f3bbe67ef3f 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -23,38 +23,459 @@ To check for security updates, go to [Security announcements for the Elastic sta ## 9.0.0 [elasticsearch-900-release-notes] +### Highlights [elasticsearch-900-highlights] + +::::{dropdown} Add new experimental `rank_vectors` mapping for late-interaction second order ranking +Late-interaction models are powerful rerankers. While their size and overall cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali. + +For more information, check [PR #118804](https://github.com/elastic/elasticsearch/pull/118804). +:::: + +::::{dropdown} Enable LOOKUP JOIN in non-snapshot builds +This effectively releases LOOKUP JOIN into tech preview. Docs will +follow in a separate PR. + +- Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds. +- Remove the grammar for the unsupported `| JOIN ...` command (without `LOOKUP` as first keyword). The way the lexer modes work, otherwise we'd also have to enable `| JOIN ...` syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages. +- Remove grammar for `LOOKUP JOIN index AS ...` because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries. + +Also fix https://github.com/elastic/elasticsearch/issues/121185 + +For more information, check [PR #121193](https://github.com/elastic/elasticsearch/pull/121193). +:::: + ### Features and enhancements [elasticsearch-900-features-enhancements] Allocation: * Add `FailedShardEntry` info to shard-failed task source string [#125520](https://github.com/elastic/elasticsearch/pull/125520) (issue: [#125520](https://github.com/elastic/elasticsearch/pull/125520)) +* Add a not-master state for desired balance [#116904](https://github.com/elastic/elasticsearch/pull/116904) +* Only publish desired balance gauges on master [#115383](https://github.com/elastic/elasticsearch/pull/115383) +* Reset relocation/allocation failure counter on node join/shutdown [#119968](https://github.com/elastic/elasticsearch/pull/119968) + +Authentication: +* Allow `SSHA-256` for API key credential hash [#120997](https://github.com/elastic/elasticsearch/pull/120997) + +Authorization: +* Allow kibana_system user to manage .reindexed-v8-internal.alerts indices [#118959](https://github.com/elastic/elasticsearch/pull/118959) +* Grant necessary Kibana application privileges to `reporting_user` role [#118058](https://github.com/elastic/elasticsearch/pull/118058) +* Make reserved built-in roles queryable [#117581](https://github.com/elastic/elasticsearch/pull/117581) +* [Security Solution] Add `create_index` to `kibana_system` role for index/DS `.logs-endpoint.action.responses-*` [#115241](https://github.com/elastic/elasticsearch/pull/115241) +* [Security Solution] allows `kibana_system` user to manage .reindexed-v8-* Security Solution indices [#119054](https://github.com/elastic/elasticsearch/pull/119054) + +CCS: +* Resolve/cluster allows querying for cluster info only (no index expression required) [#119898](https://github.com/elastic/elasticsearch/pull/119898) + +CRUD: +* Metrics for indexing failures due to version conflicts [#119067](https://github.com/elastic/elasticsearch/pull/119067) +* Remove INDEX_REFRESH_BLOCK after index becomes searchable [#120807](https://github.com/elastic/elasticsearch/pull/120807) +* Suppress merge-on-recovery for older indices [#113462](https://github.com/elastic/elasticsearch/pull/113462) + +Cluster Coordination: +* Include `clusterApplyListener` in long cluster apply warnings [#120087](https://github.com/elastic/elasticsearch/pull/120087) + +Data streams: +* Add action to create index from a source index [#118890](https://github.com/elastic/elasticsearch/pull/118890) +* Add index and reindex request settings to speed up reindex [#119780](https://github.com/elastic/elasticsearch/pull/119780) +* Add rest endpoint for `create_from_source_index` [#119250](https://github.com/elastic/elasticsearch/pull/119250) +* Add sanity check to `ReindexDatastreamIndexAction` [#120231](https://github.com/elastic/elasticsearch/pull/120231) +* Adding a migration reindex cancel API [#118291](https://github.com/elastic/elasticsearch/pull/118291) +* Adding get migration reindex status [#118267](https://github.com/elastic/elasticsearch/pull/118267) +* Consistent mapping for OTel log and event bodies [#120547](https://github.com/elastic/elasticsearch/pull/120547) +* Filter deprecated settings when making dest index [#120163](https://github.com/elastic/elasticsearch/pull/120163) +* Ignore closed indices for reindex [#120244](https://github.com/elastic/elasticsearch/pull/120244) +* Improve how reindex data stream index action handles api blocks [#120084](https://github.com/elastic/elasticsearch/pull/120084) +* Initial work on `ReindexDatastreamIndexAction` [#116996](https://github.com/elastic/elasticsearch/pull/116996) +* Make `requests_per_second` configurable to throttle reindexing [#120207](https://github.com/elastic/elasticsearch/pull/120207) +* Optimized index sorting for OTel logs [#119504](https://github.com/elastic/elasticsearch/pull/119504) +* Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As A Warning [#120629](https://github.com/elastic/elasticsearch/pull/120629) +* Update data stream deprecations warnings to new format and filter searchable snapshots from response [#118562](https://github.com/elastic/elasticsearch/pull/118562) + +Distributed: +* Make various alias retrieval APIs wait for cluster to unblock [#117230](https://github.com/elastic/elasticsearch/pull/117230) +* Metrics for incremental bulk splits [#116765](https://github.com/elastic/elasticsearch/pull/116765) +* Use Azure blob batch API to delete blobs in batches [#114566](https://github.com/elastic/elasticsearch/pull/114566) + +EQL: +* Add support for partial shard results [#116388](https://github.com/elastic/elasticsearch/pull/116388) +* Optional named arguments for function in map [#118619](https://github.com/elastic/elasticsearch/pull/118619) ES|QL: +* Add ES|QL cross-cluster query telemetry collection [#119474](https://github.com/elastic/elasticsearch/pull/119474) +* Add a `LicenseAware` interface for licensed Nodes [#118931](https://github.com/elastic/elasticsearch/pull/118931) (issue: [#118931](https://github.com/elastic/elasticsearch/pull/118931)) +* Add a `PostAnalysisAware,` distribute verification [#119798](https://github.com/elastic/elasticsearch/pull/119798) +* Add a standard deviation aggregating function: STD_DEV [#116531](https://github.com/elastic/elasticsearch/pull/116531) +* Add cluster level reduction [#117731](https://github.com/elastic/elasticsearch/pull/117731) +* Add nulls support to Categorize [#117655](https://github.com/elastic/elasticsearch/pull/117655) +* Async search responses have CCS metadata while searches are running [#117265](https://github.com/elastic/elasticsearch/pull/117265) +* Check for early termination in Driver [#118188](https://github.com/elastic/elasticsearch/pull/118188) +* Do not serialize `EsIndex` in plan [#119580](https://github.com/elastic/elasticsearch/pull/119580) +* ESQL - Add Match function options [#120360](https://github.com/elastic/elasticsearch/pull/120360) +* ESQL - Allow full text functions disjunctions for non-full text functions [#120291](https://github.com/elastic/elasticsearch/pull/120291) +* ESQL - Remove restrictions for disjunctions in full text functions [#118544](https://github.com/elastic/elasticsearch/pull/118544) +* ESQL - enabling scoring with METADATA `_score` [#113120](https://github.com/elastic/elasticsearch/pull/113120) +* ESQL Add esql hash function [#117989](https://github.com/elastic/elasticsearch/pull/117989) +* ESQL Support IN operator for Date nanos [#119772](https://github.com/elastic/elasticsearch/pull/119772) (issue: [#119772](https://github.com/elastic/elasticsearch/pull/119772)) +* ESQL: CATEGORIZE as a `BlockHash` [#114317](https://github.com/elastic/elasticsearch/pull/114317) +* ESQL: Enable async get to support formatting [#111104](https://github.com/elastic/elasticsearch/pull/111104) (issue: [#111104](https://github.com/elastic/elasticsearch/pull/111104)) +* ESQL: Enterprise license enforcement for CCS [#118102](https://github.com/elastic/elasticsearch/pull/118102) +* ES|QL: Partial result on demand for async queries [#118122](https://github.com/elastic/elasticsearch/pull/118122) +* Enable KQL function as a tech preview [#119730](https://github.com/elastic/elasticsearch/pull/119730) +* Enable LOOKUP JOIN in non-snapshot builds [#121193](https://github.com/elastic/elasticsearch/pull/121193) (issue: [#121193](https://github.com/elastic/elasticsearch/pull/121193)) +* Enable node-level reduction by default [#119621](https://github.com/elastic/elasticsearch/pull/119621) +* Enable physical plan verification [#118114](https://github.com/elastic/elasticsearch/pull/118114) +* Ensure cluster string could be quoted [#120355](https://github.com/elastic/elasticsearch/pull/120355) +* Esql - Support date nanos in date extract function [#120727](https://github.com/elastic/elasticsearch/pull/120727) (issue: [#120727](https://github.com/elastic/elasticsearch/pull/120727)) +* Esql - support date nanos in date format function [#120143](https://github.com/elastic/elasticsearch/pull/120143) (issue: [#120143](https://github.com/elastic/elasticsearch/pull/120143)) +* Esql Support date nanos on date diff function [#120645](https://github.com/elastic/elasticsearch/pull/120645) (issue: [#120645](https://github.com/elastic/elasticsearch/pull/120645)) +* Esql bucket function for date nanos [#118474](https://github.com/elastic/elasticsearch/pull/118474) (issue: [#118474](https://github.com/elastic/elasticsearch/pull/118474)) +* Esql compare nanos and millis [#118027](https://github.com/elastic/elasticsearch/pull/118027) (issue: [#118027](https://github.com/elastic/elasticsearch/pull/118027)) +* Esql implicit casting for date nanos [#118697](https://github.com/elastic/elasticsearch/pull/118697) (issue: [#118697](https://github.com/elastic/elasticsearch/pull/118697)) +* Expand type compatibility for match function and operator [#117555](https://github.com/elastic/elasticsearch/pull/117555) +* Extend `TranslationAware` to all pushable expressions [#120192](https://github.com/elastic/elasticsearch/pull/120192) * Fix sorting when `aggregate_metric_double` present [#125191](https://github.com/elastic/elasticsearch/pull/125191) +* Hash functions [#118938](https://github.com/elastic/elasticsearch/pull/118938) +* Implement a `MetricsAware` interface [#121074](https://github.com/elastic/elasticsearch/pull/121074) * Infer the score mode to use from the Lucene collector [#125930](https://github.com/elastic/elasticsearch/pull/125930) +* LOOKUP JOIN using field-caps for field mapping [#117246](https://github.com/elastic/elasticsearch/pull/117246) +* Lookup join on multiple join fields not yet supported [#118858](https://github.com/elastic/elasticsearch/pull/118858) +* Move scoring in ES|QL out of snapshot [#120354](https://github.com/elastic/elasticsearch/pull/120354) +* Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` [#119889](https://github.com/elastic/elasticsearch/pull/119889) +* Push down filter passed lookup join [#118410](https://github.com/elastic/elasticsearch/pull/118410) +* Resume Driver on cancelled or early finished [#120020](https://github.com/elastic/elasticsearch/pull/120020) +* Rewrite TO_UPPER/TO_LOWER comparisons [#118870](https://github.com/elastic/elasticsearch/pull/118870) (issue: [#118870](https://github.com/elastic/elasticsearch/pull/118870)) +* ST_EXTENT aggregation [#117451](https://github.com/elastic/elasticsearch/pull/117451) (issue: [#117451](https://github.com/elastic/elasticsearch/pull/117451)) +* ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape [#118802](https://github.com/elastic/elasticsearch/pull/118802) +* Smarter field caps with subscribable listener [#116755](https://github.com/elastic/elasticsearch/pull/116755) +* Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions [#116964](https://github.com/elastic/elasticsearch/pull/116964) (issue: [#116964](https://github.com/elastic/elasticsearch/pull/116964)) +* Support partial sort fields in TopN pushdown [#116043](https://github.com/elastic/elasticsearch/pull/116043) (issue: [#116043](https://github.com/elastic/elasticsearch/pull/116043)) +* Support some stats on aggregate_metric_double [#120343](https://github.com/elastic/elasticsearch/pull/120343) (issue: [#120343](https://github.com/elastic/elasticsearch/pull/120343)) * Take double parameter markers for identifiers out of snapshot [#125690](https://github.com/elastic/elasticsearch/pull/125690) +* Take named parameters for identifier and pattern out of snapshot [#121850](https://github.com/elastic/elasticsearch/pull/121850) +* Term query for ES|QL [#117359](https://github.com/elastic/elasticsearch/pull/117359) +* Update grammar to rely on `indexPattern` instead of identifier in join target [#120494](https://github.com/elastic/elasticsearch/pull/120494) +* `_score` should not be a reserved attribute in ES|QL [#118435](https://github.com/elastic/elasticsearch/pull/118435) (issue: [#118435](https://github.com/elastic/elasticsearch/pull/118435)) + +Engine: +* Defer unpromotable shard refreshes until index refresh blocks are cleared [#120642](https://github.com/elastic/elasticsearch/pull/120642) +* POC mark read-only [#119743](https://github.com/elastic/elasticsearch/pull/119743) + +Experiences: +* Integrate IBM watsonx to Inference API for re-ranking task [#117176](https://github.com/elastic/elasticsearch/pull/117176) + +Extract&Transform: +* [Connector API] Support hard deletes with new URL param in delete endpoint [#120200](https://github.com/elastic/elasticsearch/pull/120200) +* [Connector API] Support soft-deletes of connectors [#118669](https://github.com/elastic/elasticsearch/pull/118669) +* [Connector APIs] Enforce index prefix for managed connectors [#117778](https://github.com/elastic/elasticsearch/pull/117778) + +Geo: +* Optimize indexing points with index and doc values set to true [#120271](https://github.com/elastic/elasticsearch/pull/120271) + +Health: +* Increase `replica_unassigned_buffer_time` default from 3s to 5s [#112834](https://github.com/elastic/elasticsearch/pull/112834) + +Highlighting: +* Add Highlighter for Semantic Text Fields [#118064](https://github.com/elastic/elasticsearch/pull/118064) ILM+SLM: +* Add a `replicate_for` option to the ILM `searchable_snapshot` action [#119003](https://github.com/elastic/elasticsearch/pull/119003) * Optimize usage calculation in ILM policies retrieval API [#106953](https://github.com/elastic/elasticsearch/pull/106953) (issue: [#106953](https://github.com/elastic/elasticsearch/pull/106953)) +Indices APIs: +* Add `remove_index_block` arg to `_create_from` api [#120548](https://github.com/elastic/elasticsearch/pull/120548) +* Remove index blocks by default in `create_from` [#120643](https://github.com/elastic/elasticsearch/pull/120643) +* Run `TransportGetComponentTemplateAction` on local node [#116868](https://github.com/elastic/elasticsearch/pull/116868) +* Run `TransportGetComposableIndexTemplate` on local node [#119830](https://github.com/elastic/elasticsearch/pull/119830) +* Run `TransportGetIndexTemplateAction` on local node [#119837](https://github.com/elastic/elasticsearch/pull/119837) +* introduce new categories for deprecated resources in deprecation API [#120505](https://github.com/elastic/elasticsearch/pull/120505) + +Inference: +* Add version prefix to Inference Service API path [#117095](https://github.com/elastic/elasticsearch/pull/117095) +* Remove Elastic Inference Service feature flag and deprecated setting [#120842](https://github.com/elastic/elasticsearch/pull/120842) +* Update sparse text embeddings API route for Inference Service [#118025](https://github.com/elastic/elasticsearch/pull/118025) +* [Elastic Inference Service] Add ElasticInferenceService Unified ChatCompletions Integration [#118871](https://github.com/elastic/elasticsearch/pull/118871) + +Infra/CLI: +* Strengthen encryption for elasticsearch-keystore tool to AES 256 [#119749](https://github.com/elastic/elasticsearch/pull/119749) + +Infra/Circuit Breakers: +* Add link to Circuit Breaker "Data too large" exception message [#113561](https://github.com/elastic/elasticsearch/pull/113561) + +Infra/Core: +* Add support for specifying reindexing script for system index migration [#119001](https://github.com/elastic/elasticsearch/pull/119001) +* Bump major version for feature migration system indices [#117243](https://github.com/elastic/elasticsearch/pull/117243) +* Change default Docker image to be based on UBI minimal instead of Ubuntu [#116739](https://github.com/elastic/elasticsearch/pull/116739) +* Infrastructure for assuming cluster features in the next major version [#118143](https://github.com/elastic/elasticsearch/pull/118143) +* Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#124865](https://github.com/elastic/elasticsearch/pull/124865) +* Update ASM 9.7 -> 9.7.1 to support JDK 24 [#118094](https://github.com/elastic/elasticsearch/pull/118094) + +Infra/Metrics: +* Add `ensureGreen` test method for use with `adminClient` [#113425](https://github.com/elastic/elasticsearch/pull/113425) + +Infra/REST API: +* A new query parameter `?include_source_on_error` was added for create / index, update and bulk REST APIs to control +if to include the document source in the error response in case of parsing errors. The default value is `true`. [#120725](https://github.com/elastic/elasticsearch/pull/120725) + +Infra/Scripting: +* Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts [#114002](https://github.com/elastic/elasticsearch/pull/114002) + +Infra/Settings: +* Introduce `IndexSettingDeprecatedInV8AndRemovedInV9` Setting property [#120334](https://github.com/elastic/elasticsearch/pull/120334) +* Run `TransportClusterGetSettingsAction` on local node [#119831](https://github.com/elastic/elasticsearch/pull/119831) + +Ingest Node: +* Optimize `IngestCtxMap` construction [#120833](https://github.com/elastic/elasticsearch/pull/120833) +* Optimize `IngestDocMetadata` `isAvailable` [#120753](https://github.com/elastic/elasticsearch/pull/120753) +* Optimize `IngestDocument` `FieldPath` allocation [#120573](https://github.com/elastic/elasticsearch/pull/120573) +* Optimize some per-document hot paths in the geoip processor [#120824](https://github.com/elastic/elasticsearch/pull/120824) +* Returning ignored fields in the simulate ingest API [#117214](https://github.com/elastic/elasticsearch/pull/117214) +* Run `GetPipelineTransportAction` on local node [#120445](https://github.com/elastic/elasticsearch/pull/120445) +* Run `TransportGetEnrichPolicyAction` on local node [#121124](https://github.com/elastic/elasticsearch/pull/121124) +* Run template simulation actions on local node [#120038](https://github.com/elastic/elasticsearch/pull/120038) + +License: +* Bump `TrialLicenseVersion` to allow starting new trial on 9.0 [#120198](https://github.com/elastic/elasticsearch/pull/120198) + +Logs: +* Add LogsDB option to route on sort fields [#116687](https://github.com/elastic/elasticsearch/pull/116687) +* Add a new index setting to skip recovery source when synthetic source is enabled [#114618](https://github.com/elastic/elasticsearch/pull/114618) +* Configure index sorting through index settings for logsdb [#118968](https://github.com/elastic/elasticsearch/pull/118968) (issue: [#118968](https://github.com/elastic/elasticsearch/pull/118968)) +* Optimize loading mappings when determining synthetic source usage and whether host.name can be sorted on. [#120055](https://github.com/elastic/elasticsearch/pull/120055) + +Machine Learning: +* Add DeBERTa-V2/V3 tokenizer [#111852](https://github.com/elastic/elasticsearch/pull/111852) +* Add Inference Unified API for chat completions for OpenAI [#117589](https://github.com/elastic/elasticsearch/pull/117589) +* Add Jina AI API to do inference for Embedding and Rerank models [#118652](https://github.com/elastic/elasticsearch/pull/118652) +* Add enterprise license check for Inference API actions [#119893](https://github.com/elastic/elasticsearch/pull/119893) +* Adding chunking settings to `IbmWatsonxService` [#114914](https://github.com/elastic/elasticsearch/pull/114914) +* Adding default endpoint for Elastic Rerank [#117939](https://github.com/elastic/elasticsearch/pull/117939) +* Adding endpoint creation validation for all task types to remaining services [#115020](https://github.com/elastic/elasticsearch/pull/115020) +* Automatically rollover legacy .ml-anomalies indices [#120913](https://github.com/elastic/elasticsearch/pull/120913) +* Automatically rollover legacy ml indices [#120405](https://github.com/elastic/elasticsearch/pull/120405) +* Change the auditor to write via an alias [#120064](https://github.com/elastic/elasticsearch/pull/120064) +* Check for presence of error object when validating streaming responses from integrations in the inference API [#118375](https://github.com/elastic/elasticsearch/pull/118375) +* ES|QL categorize with multiple groupings [#118173](https://github.com/elastic/elasticsearch/pull/118173) +* Ignore failures from renormalizing buckets in read-only index [#118674](https://github.com/elastic/elasticsearch/pull/118674) +* Inference duration and error metrics [#115876](https://github.com/elastic/elasticsearch/pull/115876) +* Migrate stream to core error parsing [#120722](https://github.com/elastic/elasticsearch/pull/120722) +* Remove all mentions of eis and gateway and deprecate flags that do [#116692](https://github.com/elastic/elasticsearch/pull/116692) +* Remove deprecated sort from reindex operation within dataframe analytics procedure [#117606](https://github.com/elastic/elasticsearch/pull/117606) +* Retry on `ClusterBlockException` on transform destination index [#118194](https://github.com/elastic/elasticsearch/pull/118194) +* Support mTLS for the Elastic Inference Service integration inside the inference API [#119679](https://github.com/elastic/elasticsearch/pull/119679) +* [Inference API] Add node-local rate limiting for the inference API [#120400](https://github.com/elastic/elasticsearch/pull/120400) +* [Inference API] fix spell words: covertToString to convertToString [#119922](https://github.com/elastic/elasticsearch/pull/119922) + +Mapping: +* Add Optional Source Filtering to Source Loaders [#113827](https://github.com/elastic/elasticsearch/pull/113827) +* Add option to store `sparse_vector` outside `_source` [#117917](https://github.com/elastic/elasticsearch/pull/117917) + +Network: +* Allow http unsafe buffers by default [#116115](https://github.com/elastic/elasticsearch/pull/116115) +* Http stream activity tracker and exceptions handling [#119564](https://github.com/elastic/elasticsearch/pull/119564) +* Remove HTTP content copies [#117303](https://github.com/elastic/elasticsearch/pull/117303) +* `ConnectTransportException` returns retryable BAD_GATEWAY [#118681](https://github.com/elastic/elasticsearch/pull/118681) (issue: [#118681](https://github.com/elastic/elasticsearch/pull/118681)) + +Ranking: +* Add a generic `rescorer` retriever based on the search request's rescore functionality [#118585](https://github.com/elastic/elasticsearch/pull/118585) (issue: [#118585](https://github.com/elastic/elasticsearch/pull/118585)) +* Set default reranker for text similarity reranker to Elastic reranker [#120551](https://github.com/elastic/elasticsearch/pull/120551) + +Recovery: +* Allow archive and searchable snapshots indices in N-2 version [#118941](https://github.com/elastic/elasticsearch/pull/118941) +* Trigger merges after recovery [#113102](https://github.com/elastic/elasticsearch/pull/113102) + +Reindex: +* Change Reindexing metrics unit from millis to seconds [#115721](https://github.com/elastic/elasticsearch/pull/115721) + +Relevance: +* Add Multi-Field Support for Semantic Text Fields [#120128](https://github.com/elastic/elasticsearch/pull/120128) + Search: +* Add match support for `semantic_text` fields [#117839](https://github.com/elastic/elasticsearch/pull/117839) +* Add support for `sparse_vector` queries against `semantic_text` fields [#118617](https://github.com/elastic/elasticsearch/pull/118617) +* Add support for knn vector queries on `semantic_text` fields [#119011](https://github.com/elastic/elasticsearch/pull/119011) +* Adding linear retriever to support weighted sums of sub-retrievers [#120222](https://github.com/elastic/elasticsearch/pull/120222) +* Address and remove any references of RestApiVersion version 7 [#117572](https://github.com/elastic/elasticsearch/pull/117572) +* Feat: add a user-configurable timeout parameter to the `_resolve/cluster` API [#120542](https://github.com/elastic/elasticsearch/pull/120542) * Introduce batched query execution and data-node side reduce [#121885](https://github.com/elastic/elasticsearch/pull/121885) +* Make semantic text part of the text family [#119792](https://github.com/elastic/elasticsearch/pull/119792) +* Only aggregations require at least one shard request [#115314](https://github.com/elastic/elasticsearch/pull/115314) +* Prevent data nodes from sending stack traces to coordinator when `error_trace=false` [#118266](https://github.com/elastic/elasticsearch/pull/118266) +* Propagate status codes from shard failures appropriately [#118016](https://github.com/elastic/elasticsearch/pull/118016) (issue: [#118016](https://github.com/elastic/elasticsearch/pull/118016)) +* Upgrade to Lucene 10 [#114741](https://github.com/elastic/elasticsearch/pull/114741) +* Upgrade to Lucene 10.1.0 [#119308](https://github.com/elastic/elasticsearch/pull/119308) + +Security: +* Add refresh `.security` index call between security migrations [#114879](https://github.com/elastic/elasticsearch/pull/114879) + +Snapshot/Restore: +* Add IMDSv2 support to `repository-s3` [#117748](https://github.com/elastic/elasticsearch/pull/117748) (issue: [#117748](https://github.com/elastic/elasticsearch/pull/117748)) +* Expose operation and request counts separately in repository stats [#117530](https://github.com/elastic/elasticsearch/pull/117530) (issue: [#117530](https://github.com/elastic/elasticsearch/pull/117530)) +* Retry `S3BlobContainer#getRegister` on all exceptions [#114813](https://github.com/elastic/elasticsearch/pull/114813) +* Retry internally when CAS upload is throttled [GCS] [#120250](https://github.com/elastic/elasticsearch/pull/120250) (issue: [#120250](https://github.com/elastic/elasticsearch/pull/120250)) +* Track shard snapshot progress during node shutdown [#112567](https://github.com/elastic/elasticsearch/pull/112567) + +Suggesters: +* Extensible Completion Postings Formats [#111494](https://github.com/elastic/elasticsearch/pull/111494) + +TSDB: +* Increase field limit for OTel metrics to 10 000 [#120591](https://github.com/elastic/elasticsearch/pull/120591) + +Transform: +* Add support for `extended_stats` [#120340](https://github.com/elastic/elasticsearch/pull/120340) +* Auto-migrate `max_page_search_size` [#119348](https://github.com/elastic/elasticsearch/pull/119348) +* Create upgrade mode [#117858](https://github.com/elastic/elasticsearch/pull/117858) +* Wait while index is blocked [#119542](https://github.com/elastic/elasticsearch/pull/119542) +* [Deprecation] Add `transform_ids` to outdated index [#120821](https://github.com/elastic/elasticsearch/pull/120821) + +Vector Search: +* Add new experimental `rank_vectors` mapping for late-interaction second order ranking [#118804](https://github.com/elastic/elasticsearch/pull/118804) +* Even better(er) binary quantization [#117994](https://github.com/elastic/elasticsearch/pull/117994) +* KNN vector rescoring for quantized vectors [#116663](https://github.com/elastic/elasticsearch/pull/116663) +* Mark bbq indices as GA and add rolling upgrade integration tests [#121105](https://github.com/elastic/elasticsearch/pull/121105) +* Speed up bit compared with floats or bytes script operations [#117199](https://github.com/elastic/elasticsearch/pull/117199) ### Fixes [elasticsearch-900-fixes] +Aggregations: +* Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations [#116174](https://github.com/elastic/elasticsearch/pull/116174) (issue: [#116174](https://github.com/elastic/elasticsearch/pull/116174)) + +Analysis: +* Analyze API to return 400 for wrong custom analyzer [#121568](https://github.com/elastic/elasticsearch/pull/121568) (issue: [#121568](https://github.com/elastic/elasticsearch/pull/121568)) + +Authentication: +* Fix NPE for missing Content Type header in OIDC Authenticator [#126191](https://github.com/elastic/elasticsearch/pull/126191) + +CAT APIs: +* Fix cat_component_templates documentation [#120487](https://github.com/elastic/elasticsearch/pull/120487) + +CRUD: +* Preserve thread context when waiting for segment generation in RTG [#114623](https://github.com/elastic/elasticsearch/pull/114623) +* Preserve thread context when waiting for segment generation in RTG [#117148](https://github.com/elastic/elasticsearch/pull/117148) + Data streams: +* Avoid updating settings version in `MetadataMigrateToDataStreamService` when settings have not changed [#118704](https://github.com/elastic/elasticsearch/pull/118704) +* Block-writes cannot be added after read-only [#119007](https://github.com/elastic/elasticsearch/pull/119007) (issue: [#119007](https://github.com/elastic/elasticsearch/pull/119007)) +* Ensure removal of index blocks does not leave key with null value [#122246](https://github.com/elastic/elasticsearch/pull/122246) * Fixes a invalid warning from being issued when restoring a system data stream from a snapshot. [#125881](https://github.com/elastic/elasticsearch/pull/125881) +* Match dot prefix of migrated DS backing index with the source index [#120042](https://github.com/elastic/elasticsearch/pull/120042) +* Refresh source index before reindexing data stream index [#120752](https://github.com/elastic/elasticsearch/pull/120752) (issue: [#120752](https://github.com/elastic/elasticsearch/pull/120752)) +* `ReindexDataStreamIndex` bug in assertion caused by reference equality [#121325](https://github.com/elastic/elasticsearch/pull/121325) + +Downsampling: +* Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` [#121727](https://github.com/elastic/elasticsearch/pull/121727) (issues: [#121727](https://github.com/elastic/elasticsearch/pull/121727), [#121727](https://github.com/elastic/elasticsearch/pull/121727)) ES|QL: +* Allow the data type of `null` in filters [#118324](https://github.com/elastic/elasticsearch/pull/118324) (issue: [#118324](https://github.com/elastic/elasticsearch/pull/118324)) +* Correct line and column numbers of missing named parameters [#120852](https://github.com/elastic/elasticsearch/pull/120852) +* Drop null columns in text formats [#117643](https://github.com/elastic/elasticsearch/pull/117643) (issue: [#117643](https://github.com/elastic/elasticsearch/pull/117643)) +* Fix ROUND() with unsigned longs throwing in some edge cases [#119536](https://github.com/elastic/elasticsearch/pull/119536) +* Fix TDigestState.read CB leaks [#114303](https://github.com/elastic/elasticsearch/pull/114303) (issue: [#114303](https://github.com/elastic/elasticsearch/pull/114303)) +* Fix TopN row size estimate [#119476](https://github.com/elastic/elasticsearch/pull/119476) (issue: [#119476](https://github.com/elastic/elasticsearch/pull/119476)) +* Fix `AbstractShapeGeometryFieldMapperTests` [#119265](https://github.com/elastic/elasticsearch/pull/119265) (issue: [#119265](https://github.com/elastic/elasticsearch/pull/119265)) * Fix `ReplaceMissingFieldsWithNull` [#125764](https://github.com/elastic/elasticsearch/pull/125764) (issues: [#125764](https://github.com/elastic/elasticsearch/pull/125764), [#125764](https://github.com/elastic/elasticsearch/pull/125764), [#125764](https://github.com/elastic/elasticsearch/pull/125764)) +* Fix a bug in TOP [#121552](https://github.com/elastic/elasticsearch/pull/121552) +* Fix async stop sometimes not properly collecting result [#121843](https://github.com/elastic/elasticsearch/pull/121843) (issue: [#121843](https://github.com/elastic/elasticsearch/pull/121843)) +* Fix attribute set equals [#118823](https://github.com/elastic/elasticsearch/pull/118823) +* Fix double lookup failure on ESQL [#115616](https://github.com/elastic/elasticsearch/pull/115616) (issue: [#115616](https://github.com/elastic/elasticsearch/pull/115616)) +* Fix queries with document level security on lookup indexes [#120617](https://github.com/elastic/elasticsearch/pull/120617) (issue: [#120617](https://github.com/elastic/elasticsearch/pull/120617)) +* Fix writing for LOOKUP status [#119296](https://github.com/elastic/elasticsearch/pull/119296) (issue: [#119296](https://github.com/elastic/elasticsearch/pull/119296)) +* Limit memory usage of `fold` [#118602](https://github.com/elastic/elasticsearch/pull/118602) +* Limit size of query [#117898](https://github.com/elastic/elasticsearch/pull/117898) +* Remove redundant sorts from execution plan [#121156](https://github.com/elastic/elasticsearch/pull/121156) +* Revert unwanted ES|QL lexer changes from PR #120354 [#120538](https://github.com/elastic/elasticsearch/pull/120538) + +Health: +* Do not recommend increasing `max_shards_per_node` [#120458](https://github.com/elastic/elasticsearch/pull/120458) Indices APIs: +* Add `?master_timeout` to `POST /_ilm/migrate_to_data_tiers` [#120883](https://github.com/elastic/elasticsearch/pull/120883) +* Fix broken yaml test `30_create_from` [#120662](https://github.com/elastic/elasticsearch/pull/120662) +* Include hidden indices in `DeprecationInfoAction` [#118035](https://github.com/elastic/elasticsearch/pull/118035) (issue: [#118035](https://github.com/elastic/elasticsearch/pull/118035)) * Preventing `ConcurrentModificationException` when updating settings for more than one index [#126077](https://github.com/elastic/elasticsearch/pull/126077) +Inference: +* [Inference API] Put back legacy EIS URL setting [#121207](https://github.com/elastic/elasticsearch/pull/121207) + +Infra/Core: +* Epoch Millis Rounding Down and Not Up 2 [#118353](https://github.com/elastic/elasticsearch/pull/118353) +* System data streams are not being upgraded in the feature migration API [#124884](https://github.com/elastic/elasticsearch/pull/124884) (issue: [#124884](https://github.com/elastic/elasticsearch/pull/124884)) +* Wrap jackson exception on malformed json string [#114445](https://github.com/elastic/elasticsearch/pull/114445) (issue: [#114445](https://github.com/elastic/elasticsearch/pull/114445)) + +Infra/Logging: +* Move `SlowLogFieldProvider` instantiation to node construction [#117949](https://github.com/elastic/elasticsearch/pull/117949) + +Infra/Metrics: +* Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]` [#114177](https://github.com/elastic/elasticsearch/pull/114177) + +Infra/Plugins: +* Remove unnecessary entitlement [#120959](https://github.com/elastic/elasticsearch/pull/120959) +* Restrict agent entitlements to the system classloader unnamed module [#120546](https://github.com/elastic/elasticsearch/pull/120546) + +Infra/REST API: +* Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null. [#113413](https://github.com/elastic/elasticsearch/pull/113413) (issue: [#113413](https://github.com/elastic/elasticsearch/pull/113413)) + +Infra/Scripting: +* Register mustache size limit setting [#119291](https://github.com/elastic/elasticsearch/pull/119291) + +Infra/Settings: +* Don't allow secure settings in YML config (109115) [#115779](https://github.com/elastic/elasticsearch/pull/115779) (issue: [#115779](https://github.com/elastic/elasticsearch/pull/115779)) + +Ingest Node: +* Add warning headers for ingest pipelines containing special characters [#114837](https://github.com/elastic/elasticsearch/pull/114837) (issue: [#114837](https://github.com/elastic/elasticsearch/pull/114837)) +* Fix geoip databases index access after system feature migration [#121196](https://github.com/elastic/elasticsearch/pull/121196) + +Logs: +* Always check if index mode is logsdb [#116922](https://github.com/elastic/elasticsearch/pull/116922) + Machine Learning: +* Change format for Unified Chat [#121396](https://github.com/elastic/elasticsearch/pull/121396) +* Fix get all inference endponts not returning multiple endpoints sharing model deployment [#121821](https://github.com/elastic/elasticsearch/pull/121821) +* Fixing bedrock event executor terminated cache issue [#118177](https://github.com/elastic/elasticsearch/pull/118177) (issue: [#118177](https://github.com/elastic/elasticsearch/pull/118177)) +* Fixing bug setting index when parsing Google Vertex AI results [#117287](https://github.com/elastic/elasticsearch/pull/117287) * Prevent get datafeeds stats API returning an error when local tasks are slow to stop [#125477](https://github.com/elastic/elasticsearch/pull/125477) (issue: [#125477](https://github.com/elastic/elasticsearch/pull/125477)) +* Updating Inference Update API documentation to have the correct PUT method [#121048](https://github.com/elastic/elasticsearch/pull/121048) +* Wait for up to 2 seconds for yellow status before starting search [#115938](https://github.com/elastic/elasticsearch/pull/115938) (issues: [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938)) +* [Inference API] Fix unique ID message for inference ID matches trained model ID [#119543](https://github.com/elastic/elasticsearch/pull/119543) (issue: [#119543](https://github.com/elastic/elasticsearch/pull/119543)) + +Mapping: +* Enable New Semantic Text Format Only On Newly Created Indices [#121556](https://github.com/elastic/elasticsearch/pull/121556) +* Fix propagation of dynamic mapping parameter when applying `copy_to` [#121109](https://github.com/elastic/elasticsearch/pull/121109) (issue: [#121109](https://github.com/elastic/elasticsearch/pull/121109)) +* Fix realtime get of nested fields with synthetic source [#119575](https://github.com/elastic/elasticsearch/pull/119575) (issue: [#119575](https://github.com/elastic/elasticsearch/pull/119575)) +* Merge field mappers when updating mappings with [subobjects:false] [#120370](https://github.com/elastic/elasticsearch/pull/120370) (issue: [#120370](https://github.com/elastic/elasticsearch/pull/120370)) +* Tweak `copy_to` handling in synthetic `_source` to account for nested objects [#120974](https://github.com/elastic/elasticsearch/pull/120974) (issue: [#120974](https://github.com/elastic/elasticsearch/pull/120974)) + +Network: +* Remove ChunkedToXContentBuilder [#119310](https://github.com/elastic/elasticsearch/pull/119310) (issue: [#119310](https://github.com/elastic/elasticsearch/pull/119310)) Search: +* Catch and handle disconnect exceptions in search [#115836](https://github.com/elastic/elasticsearch/pull/115836) +* Fix leak in `DfsQueryPhase` and introduce search disconnect stress test [#116060](https://github.com/elastic/elasticsearch/pull/116060) (issue: [#116060](https://github.com/elastic/elasticsearch/pull/116060)) +* Fix/QueryBuilderBWCIT_muted_test [#117831](https://github.com/elastic/elasticsearch/pull/117831) +* In this pr, a 400 error is returned when _source / _seq_no / _feature / _nested_path / _field_names is requested, rather a 5xx [#117229](https://github.com/elastic/elasticsearch/pull/117229) +* Inconsistency in the _analyzer api when the index is not included [#115930](https://github.com/elastic/elasticsearch/pull/115930) +* Log stack traces on data nodes before they are cleared for transport [#125732](https://github.com/elastic/elasticsearch/pull/125732) * Re-enable parallel collection for field sorted top hits [#125916](https://github.com/elastic/elasticsearch/pull/125916) +* Remove duplicate code in ESIntegTestCase [#120799](https://github.com/elastic/elasticsearch/pull/120799) +* SearchStatesIt failures reported by CI [#117618](https://github.com/elastic/elasticsearch/pull/117618) (issues: [#117618](https://github.com/elastic/elasticsearch/pull/117618), [#117618](https://github.com/elastic/elasticsearch/pull/117618)) +* Skip fetching _inference_fields field in legacy semantic_text format [#121720](https://github.com/elastic/elasticsearch/pull/121720) +* Test/107515 restore template with match only text mapper it fail [#120392](https://github.com/elastic/elasticsearch/pull/120392) (issue: [#120392](https://github.com/elastic/elasticsearch/pull/120392)) +* Updated Date Range to Follow Documentation When Assuming Missing Values [#112258](https://github.com/elastic/elasticsearch/pull/112258) (issue: [#112258](https://github.com/elastic/elasticsearch/pull/112258)) +* `CrossClusterIT` `testCancel` failure [#117750](https://github.com/elastic/elasticsearch/pull/117750) (issue: [#117750](https://github.com/elastic/elasticsearch/pull/117750)) +* `SearchServiceTests.testParseSourceValidation` failure [#117963](https://github.com/elastic/elasticsearch/pull/117963) + +Snapshot/Restore: +* Add undeclared Azure settings, modify test to exercise them [#118634](https://github.com/elastic/elasticsearch/pull/118634) +* Retry throttled snapshot deletions [#113237](https://github.com/elastic/elasticsearch/pull/113237) +* Use the system index descriptor in the snapshot blob cache cleanup task [#120937](https://github.com/elastic/elasticsearch/pull/120937) (issue: [#120937](https://github.com/elastic/elasticsearch/pull/120937)) + +Store: +* Do not capture `ClusterChangedEvent` in `IndicesStore` call to #onClusterStateShardsClosed [#120193](https://github.com/elastic/elasticsearch/pull/120193) + +Vector Search: +* Apply default k for knn query eagerly [#118774](https://github.com/elastic/elasticsearch/pull/118774) +* Fix `bbq_hnsw` merge file cleanup on random IO exceptions [#119691](https://github.com/elastic/elasticsearch/pull/119691) (issue: [#119691](https://github.com/elastic/elasticsearch/pull/119691)) + +Watcher: +* Watcher history index has too many indexed fields - [#117701](https://github.com/elastic/elasticsearch/pull/117701) (issue: [#117701](https://github.com/elastic/elasticsearch/pull/117701)) ## 9.1.0 [elasticsearch-910-release-notes] From c37ac17ab0aae3f12b1a46c00618c8e7da3f8693 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 7 Apr 2025 13:48:51 -0400 Subject: [PATCH 15/34] Remove 9.1.0 bundle for now --- docs/release-notes/breaking-changes.md | 18 +- docs/release-notes/deprecations.md | 14 +- docs/release-notes/index.md | 515 ++++++++++--------------- 3 files changed, 221 insertions(+), 326 deletions(-) diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index e71e68a74777b..9c6b3122957e3 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -37,6 +37,9 @@ will prevent node from starting [#118366](https://github.com/elastic/elasticsear Cluster Coordination: * Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903) +Discovery-Plugins: +* Upgrade `discovery-ec2` to AWS SDK v2 [#122062](https://github.com/elastic/elasticsearch/pull/122062) + EQL: * Set allow_partial_search_results=true by default [#120267](https://github.com/elastic/elasticsearch/pull/120267) @@ -60,6 +63,7 @@ Infra/Core: Infra/Logging: * Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: {es-issue}83251[#83251]) +* Rename deprecation index template [#125606](https://github.com/elastic/elasticsearch/pull/125606) (issue: {es-issue}125445[#125445]) Infra/Metrics: * Deprecated tracing.apm.* settings got removed. [#119926](https://github.com/elastic/elasticsearch/pull/119926) @@ -90,6 +94,7 @@ Snapshot/Restore: * Remove deprecated `xpack.searchable.snapshot.allocate_on_rolling_restart` setting [#114202](https://github.com/elastic/elasticsearch/pull/114202) TLS: +* Drop `TLS_RSA` cipher support for JDK 24 [#123600](https://github.com/elastic/elasticsearch/pull/123600) * Remove TLSv1.1 from default protocols [#121731](https://github.com/elastic/elasticsearch/pull/121731) Transform: @@ -102,16 +107,3 @@ Watcher: * Removing support for types field in watcher search [#120748](https://github.com/elastic/elasticsearch/pull/120748) - -## 9.1.0 [elasticsearch-910-breaking-changes] - -Discovery-Plugins: -* Upgrade `discovery-ec2` to AWS SDK v2 [#122062](https://github.com/elastic/elasticsearch/pull/122062) - -Infra/Logging: -* Rename deprecation index template [#125606](https://github.com/elastic/elasticsearch/pull/125606) (issue: {es-issue}125445[#125445]) - -TLS: -* Drop `TLS_RSA` cipher support for JDK 24 [#123600](https://github.com/elastic/elasticsearch/pull/123600) - - diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index d7d66fdd0b17d..1d353648292ec 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -25,23 +25,17 @@ Ingest Node: * Fix `_type` deprecation on simulate pipeline API [#116259](https://github.com/elastic/elasticsearch/pull/116259) Machine Learning: +* Add deprecation warning for flush API [#121667](https://github.com/elastic/elasticsearch/pull/121667) (issue: {es-issue}121506[#121506]) * Removing index alias creation for deprecated transforms notification index [#117583](https://github.com/elastic/elasticsearch/pull/117583) * [Inference API] Deprecate elser service [#113216](https://github.com/elastic/elasticsearch/pull/113216) Rollup: * Emit deprecation warning when executing one of the rollup APIs [#113131](https://github.com/elastic/elasticsearch/pull/113131) -Security: -* Deprecate certificate based remote cluster security model [#120806](https://github.com/elastic/elasticsearch/pull/120806) - - - -## 9.1.0 [elasticsearch-910-deprecations] - -Machine Learning: -* Add deprecation warning for flush API [#121667](https://github.com/elastic/elasticsearch/pull/121667) (issue: {es-issue}121506[#121506]) - Search: * Deprecate Behavioral Analytics CRUD apis [#122960](https://github.com/elastic/elasticsearch/pull/122960) +Security: +* Deprecate certificate based remote cluster security model [#120806](https://github.com/elastic/elasticsearch/pull/120806) + diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 75f3bbe67ef3f..dd3995ddea1ae 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -44,11 +44,19 @@ Also fix https://github.com/elastic/elasticsearch/issues/121185 For more information, check [PR #121193](https://github.com/elastic/elasticsearch/pull/121193). :::: +::::{dropdown} Release semantic_text as a GA feature +semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. + +For more information, check [PR #124669](https://github.com/elastic/elasticsearch/pull/124669). +:::: + ### Features and enhancements [elasticsearch-900-features-enhancements] Allocation: * Add `FailedShardEntry` info to shard-failed task source string [#125520](https://github.com/elastic/elasticsearch/pull/125520) (issue: [#125520](https://github.com/elastic/elasticsearch/pull/125520)) * Add a not-master state for desired balance [#116904](https://github.com/elastic/elasticsearch/pull/116904) +* Add cache support in `TransportGetAllocationStatsAction` [#124898](https://github.com/elastic/elasticsearch/pull/124898) (issue: [#124898](https://github.com/elastic/elasticsearch/pull/124898)) +* Introduce `AllocationBalancingRoundSummaryService` [#120957](https://github.com/elastic/elasticsearch/pull/120957) * Only publish desired balance gauges on master [#115383](https://github.com/elastic/elasticsearch/pull/115383) * Reset relocation/allocation failure counter on node join/shutdown [#119968](https://github.com/elastic/elasticsearch/pull/119968) @@ -57,6 +65,7 @@ Authentication: Authorization: * Allow kibana_system user to manage .reindexed-v8-internal.alerts indices [#118959](https://github.com/elastic/elasticsearch/pull/118959) +* Do not fetch reserved roles from native store when Get Role API is called [#121971](https://github.com/elastic/elasticsearch/pull/121971) * Grant necessary Kibana application privileges to `reporting_user` role [#118058](https://github.com/elastic/elasticsearch/pull/118058) * Make reserved built-in roles queryable [#117581](https://github.com/elastic/elasticsearch/pull/117581) * [Security Solution] Add `create_index` to `kibana_system` role for index/DS `.logs-endpoint.action.responses-*` [#115241](https://github.com/elastic/elasticsearch/pull/115241) @@ -66,6 +75,7 @@ CCS: * Resolve/cluster allows querying for cluster info only (no index expression required) [#119898](https://github.com/elastic/elasticsearch/pull/119898) CRUD: +* Enhance memory accounting for document expansion and introduce max document size limit [#123543](https://github.com/elastic/elasticsearch/pull/123543) * Metrics for indexing failures due to version conflicts [#119067](https://github.com/elastic/elasticsearch/pull/119067) * Remove INDEX_REFRESH_BLOCK after index becomes searchable [#120807](https://github.com/elastic/elasticsearch/pull/120807) * Suppress merge-on-recovery for older indices [#113462](https://github.com/elastic/elasticsearch/pull/113462) @@ -76,6 +86,7 @@ Cluster Coordination: Data streams: * Add action to create index from a source index [#118890](https://github.com/elastic/elasticsearch/pull/118890) * Add index and reindex request settings to speed up reindex [#119780](https://github.com/elastic/elasticsearch/pull/119780) +* Add index mode to get data stream API [#122486](https://github.com/elastic/elasticsearch/pull/122486) * Add rest endpoint for `create_from_source_index` [#119250](https://github.com/elastic/elasticsearch/pull/119250) * Add sanity check to `ReindexDatastreamIndexAction` [#120231](https://github.com/elastic/elasticsearch/pull/120231) * Adding a migration reindex cancel API [#118291](https://github.com/elastic/elasticsearch/pull/118291) @@ -87,7 +98,13 @@ Data streams: * Initial work on `ReindexDatastreamIndexAction` [#116996](https://github.com/elastic/elasticsearch/pull/116996) * Make `requests_per_second` configurable to throttle reindexing [#120207](https://github.com/elastic/elasticsearch/pull/120207) * Optimized index sorting for OTel logs [#119504](https://github.com/elastic/elasticsearch/pull/119504) +* Reindex data stream indices on different nodes [#125171](https://github.com/elastic/elasticsearch/pull/125171) * Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As A Warning [#120629](https://github.com/elastic/elasticsearch/pull/120629) +* Retry ILM async action after reindexing data stream [#124149](https://github.com/elastic/elasticsearch/pull/124149) +* Run `TransportGetDataStreamLifecycleAction` on local node [#125214](https://github.com/elastic/elasticsearch/pull/125214) +* Run `TransportGetDataStreamOptionsAction` on local node [#125213](https://github.com/elastic/elasticsearch/pull/125213) +* Run `TransportGetDataStreamsAction` on local node [#122852](https://github.com/elastic/elasticsearch/pull/122852) +* Set cause on create index request in create from action [#124363](https://github.com/elastic/elasticsearch/pull/124363) * Update data stream deprecations warnings to new format and filter searchable snapshots from response [#118562](https://github.com/elastic/elasticsearch/pull/118562) Distributed: @@ -95,6 +112,10 @@ Distributed: * Metrics for incremental bulk splits [#116765](https://github.com/elastic/elasticsearch/pull/116765) * Use Azure blob batch API to delete blobs in batches [#114566](https://github.com/elastic/elasticsearch/pull/114566) +Downsampling: +* Improve downsample performance by buffering docids and do bulk processing [#124477](https://github.com/elastic/elasticsearch/pull/124477) +* Improve rolling up metrics [#124739](https://github.com/elastic/elasticsearch/pull/124739) + EQL: * Add support for partial shard results [#116388](https://github.com/elastic/elasticsearch/pull/116388) * Optional named arguments for function in map [#118619](https://github.com/elastic/elasticsearch/pull/118619) @@ -105,20 +126,31 @@ ES|QL: * Add a `PostAnalysisAware,` distribute verification [#119798](https://github.com/elastic/elasticsearch/pull/119798) * Add a standard deviation aggregating function: STD_DEV [#116531](https://github.com/elastic/elasticsearch/pull/116531) * Add cluster level reduction [#117731](https://github.com/elastic/elasticsearch/pull/117731) +* Add initial grammar and changes for FORK [#121948](https://github.com/elastic/elasticsearch/pull/121948) +* Add initial grammar and planning for RRF (snapshot) [#123396](https://github.com/elastic/elasticsearch/pull/123396) * Add nulls support to Categorize [#117655](https://github.com/elastic/elasticsearch/pull/117655) +* Allow partial results in ES|QL [#121942](https://github.com/elastic/elasticsearch/pull/121942) +* Allow skip shards with `_tier` and `_index` in ES|QL [#123728](https://github.com/elastic/elasticsearch/pull/123728) * Async search responses have CCS metadata while searches are running [#117265](https://github.com/elastic/elasticsearch/pull/117265) +* Avoid `NamedWritable` in block serialization [#124394](https://github.com/elastic/elasticsearch/pull/124394) +* Calculate concurrent node limit [#124901](https://github.com/elastic/elasticsearch/pull/124901) * Check for early termination in Driver [#118188](https://github.com/elastic/elasticsearch/pull/118188) * Do not serialize `EsIndex` in plan [#119580](https://github.com/elastic/elasticsearch/pull/119580) +* Double parameter markers for identifiers [#122459](https://github.com/elastic/elasticsearch/pull/122459) * ESQL - Add Match function options [#120360](https://github.com/elastic/elasticsearch/pull/120360) * ESQL - Allow full text functions disjunctions for non-full text functions [#120291](https://github.com/elastic/elasticsearch/pull/120291) * ESQL - Remove restrictions for disjunctions in full text functions [#118544](https://github.com/elastic/elasticsearch/pull/118544) * ESQL - enabling scoring with METADATA `_score` [#113120](https://github.com/elastic/elasticsearch/pull/113120) * ESQL Add esql hash function [#117989](https://github.com/elastic/elasticsearch/pull/117989) * ESQL Support IN operator for Date nanos [#119772](https://github.com/elastic/elasticsearch/pull/119772) (issue: [#119772](https://github.com/elastic/elasticsearch/pull/119772)) +* ESQL: Align `RENAME` behavior with `EVAL` for sequential processing [#122250](https://github.com/elastic/elasticsearch/pull/122250) (issue: [#122250](https://github.com/elastic/elasticsearch/pull/122250)) * ESQL: CATEGORIZE as a `BlockHash` [#114317](https://github.com/elastic/elasticsearch/pull/114317) * ESQL: Enable async get to support formatting [#111104](https://github.com/elastic/elasticsearch/pull/111104) (issue: [#111104](https://github.com/elastic/elasticsearch/pull/111104)) * ESQL: Enterprise license enforcement for CCS [#118102](https://github.com/elastic/elasticsearch/pull/118102) +* ES|QL - Add scoring for full text functions disjunctions [#121793](https://github.com/elastic/elasticsearch/pull/121793) +* ES|QL slow log [#124094](https://github.com/elastic/elasticsearch/pull/124094) * ES|QL: Partial result on demand for async queries [#118122](https://github.com/elastic/elasticsearch/pull/118122) +* ES|QL: Support `::date` in inline cast [#123460](https://github.com/elastic/elasticsearch/pull/123460) (issue: [#123460](https://github.com/elastic/elasticsearch/pull/123460)) * Enable KQL function as a tech preview [#119730](https://github.com/elastic/elasticsearch/pull/119730) * Enable LOOKUP JOIN in non-snapshot builds [#121193](https://github.com/elastic/elasticsearch/pull/121193) (issue: [#121193](https://github.com/elastic/elasticsearch/pull/121193)) * Enable node-level reduction by default [#119621](https://github.com/elastic/elasticsearch/pull/119621) @@ -132,32 +164,51 @@ ES|QL: * Esql implicit casting for date nanos [#118697](https://github.com/elastic/elasticsearch/pull/118697) (issue: [#118697](https://github.com/elastic/elasticsearch/pull/118697)) * Expand type compatibility for match function and operator [#117555](https://github.com/elastic/elasticsearch/pull/117555) * Extend `TranslationAware` to all pushable expressions [#120192](https://github.com/elastic/elasticsearch/pull/120192) +* Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: [#123290](https://github.com/elastic/elasticsearch/pull/123290)) * Fix sorting when `aggregate_metric_double` present [#125191](https://github.com/elastic/elasticsearch/pull/125191) * Hash functions [#118938](https://github.com/elastic/elasticsearch/pull/118938) * Implement a `MetricsAware` interface [#121074](https://github.com/elastic/elasticsearch/pull/121074) +* Implement runtime skip_unavailable=true [#121240](https://github.com/elastic/elasticsearch/pull/121240) +* Include failures in partial response [#124929](https://github.com/elastic/elasticsearch/pull/124929) * Infer the score mode to use from the Lucene collector [#125930](https://github.com/elastic/elasticsearch/pull/125930) +* Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) +* Introduce `allow_partial_results` setting in ES|QL [#122890](https://github.com/elastic/elasticsearch/pull/122890) +* Introduce a pre-mapping logical plan processing step [#121260](https://github.com/elastic/elasticsearch/pull/121260) +* Keep ordinals in conversion functions [#125357](https://github.com/elastic/elasticsearch/pull/125357) * LOOKUP JOIN using field-caps for field mapping [#117246](https://github.com/elastic/elasticsearch/pull/117246) * Lookup join on multiple join fields not yet supported [#118858](https://github.com/elastic/elasticsearch/pull/118858) * Move scoring in ES|QL out of snapshot [#120354](https://github.com/elastic/elasticsearch/pull/120354) * Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` [#119889](https://github.com/elastic/elasticsearch/pull/119889) +* Pragma to load from stored fields [#122891](https://github.com/elastic/elasticsearch/pull/122891) +* Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: [#123381](https://github.com/elastic/elasticsearch/pull/123381)) * Push down filter passed lookup join [#118410](https://github.com/elastic/elasticsearch/pull/118410) +* Remove page alignment in exchange sink [#124610](https://github.com/elastic/elasticsearch/pull/124610) +* Render `aggregate_metric_double` [#122660](https://github.com/elastic/elasticsearch/pull/122660) +* Report `original_types` [#124913](https://github.com/elastic/elasticsearch/pull/124913) +* Report failures on partial results [#124823](https://github.com/elastic/elasticsearch/pull/124823) * Resume Driver on cancelled or early finished [#120020](https://github.com/elastic/elasticsearch/pull/120020) +* Retry ES|QL node requests on shard level failures [#120774](https://github.com/elastic/elasticsearch/pull/120774) +* Reuse child `outputSet` inside the plan where possible [#124611](https://github.com/elastic/elasticsearch/pull/124611) * Rewrite TO_UPPER/TO_LOWER comparisons [#118870](https://github.com/elastic/elasticsearch/pull/118870) (issue: [#118870](https://github.com/elastic/elasticsearch/pull/118870)) * ST_EXTENT aggregation [#117451](https://github.com/elastic/elasticsearch/pull/117451) (issue: [#117451](https://github.com/elastic/elasticsearch/pull/117451)) * ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape [#118802](https://github.com/elastic/elasticsearch/pull/118802) * Smarter field caps with subscribable listener [#116755](https://github.com/elastic/elasticsearch/pull/116755) * Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions [#116964](https://github.com/elastic/elasticsearch/pull/116964) (issue: [#116964](https://github.com/elastic/elasticsearch/pull/116964)) +* Support partial results in CCS in ES|QL [#122708](https://github.com/elastic/elasticsearch/pull/122708) * Support partial sort fields in TopN pushdown [#116043](https://github.com/elastic/elasticsearch/pull/116043) (issue: [#116043](https://github.com/elastic/elasticsearch/pull/116043)) * Support some stats on aggregate_metric_double [#120343](https://github.com/elastic/elasticsearch/pull/120343) (issue: [#120343](https://github.com/elastic/elasticsearch/pull/120343)) +* Support subset of metrics in aggregate metric double [#121805](https://github.com/elastic/elasticsearch/pull/121805) * Take double parameter markers for identifiers out of snapshot [#125690](https://github.com/elastic/elasticsearch/pull/125690) * Take named parameters for identifier and pattern out of snapshot [#121850](https://github.com/elastic/elasticsearch/pull/121850) * Term query for ES|QL [#117359](https://github.com/elastic/elasticsearch/pull/117359) * Update grammar to rely on `indexPattern` instead of identifier in join target [#120494](https://github.com/elastic/elasticsearch/pull/120494) +* `ToAggregateMetricDouble` function [#124595](https://github.com/elastic/elasticsearch/pull/124595) * `_score` should not be a reserved attribute in ES|QL [#118435](https://github.com/elastic/elasticsearch/pull/118435) (issue: [#118435](https://github.com/elastic/elasticsearch/pull/118435)) Engine: * Defer unpromotable shard refreshes until index refresh blocks are cleared [#120642](https://github.com/elastic/elasticsearch/pull/120642) * POC mark read-only [#119743](https://github.com/elastic/elasticsearch/pull/119743) +* Threadpool merge scheduler [#120869](https://github.com/elastic/elasticsearch/pull/120869) Experiences: * Integrate IBM watsonx to Inference API for re-ranking task [#117176](https://github.com/elastic/elasticsearch/pull/117176) @@ -171,6 +222,7 @@ Geo: * Optimize indexing points with index and doc values set to true [#120271](https://github.com/elastic/elasticsearch/pull/120271) Health: +* Add health indicator impact to `HealthPeriodicLogger` [#122390](https://github.com/elastic/elasticsearch/pull/122390) * Increase `replica_unassigned_buffer_time` default from 3s to 5s [#112834](https://github.com/elastic/elasticsearch/pull/112834) Highlighting: @@ -178,14 +230,20 @@ Highlighting: ILM+SLM: * Add a `replicate_for` option to the ILM `searchable_snapshot` action [#119003](https://github.com/elastic/elasticsearch/pull/119003) +* Improve SLM Health Indicator to cover missing snapshot [#121370](https://github.com/elastic/elasticsearch/pull/121370) * Optimize usage calculation in ILM policies retrieval API [#106953](https://github.com/elastic/elasticsearch/pull/106953) (issue: [#106953](https://github.com/elastic/elasticsearch/pull/106953)) +* Process ILM cluster state updates on another thread [#123712](https://github.com/elastic/elasticsearch/pull/123712) +* Run `TransportExplainLifecycleAction` on local node [#122885](https://github.com/elastic/elasticsearch/pull/122885) +* Truncate `step_info` and error reason in ILM execution state and history [#125054](https://github.com/elastic/elasticsearch/pull/125054) (issue: [#125054](https://github.com/elastic/elasticsearch/pull/125054)) Indices APIs: * Add `remove_index_block` arg to `_create_from` api [#120548](https://github.com/elastic/elasticsearch/pull/120548) +* Avoid creating known_fields for every check in Alias [#124690](https://github.com/elastic/elasticsearch/pull/124690) * Remove index blocks by default in `create_from` [#120643](https://github.com/elastic/elasticsearch/pull/120643) * Run `TransportGetComponentTemplateAction` on local node [#116868](https://github.com/elastic/elasticsearch/pull/116868) * Run `TransportGetComposableIndexTemplate` on local node [#119830](https://github.com/elastic/elasticsearch/pull/119830) * Run `TransportGetIndexTemplateAction` on local node [#119837](https://github.com/elastic/elasticsearch/pull/119837) +* Run `TransportGetMappingsAction` on local node [#122921](https://github.com/elastic/elasticsearch/pull/122921) * introduce new categories for deprecated resources in deprecation API [#120505](https://github.com/elastic/elasticsearch/pull/120505) Inference: @@ -193,8 +251,10 @@ Inference: * Remove Elastic Inference Service feature flag and deprecated setting [#120842](https://github.com/elastic/elasticsearch/pull/120842) * Update sparse text embeddings API route for Inference Service [#118025](https://github.com/elastic/elasticsearch/pull/118025) * [Elastic Inference Service] Add ElasticInferenceService Unified ChatCompletions Integration [#118871](https://github.com/elastic/elasticsearch/pull/118871) +* [Inference API] Rename `model_id` prop to model in EIS sparse inference request body [#122272](https://github.com/elastic/elasticsearch/pull/122272) Infra/CLI: +* Ignore _JAVA_OPTIONS [#124843](https://github.com/elastic/elasticsearch/pull/124843) * Strengthen encryption for elasticsearch-keystore tool to AES 256 [#119749](https://github.com/elastic/elasticsearch/pull/119749) Infra/Circuit Breakers: @@ -204,8 +264,11 @@ Infra/Core: * Add support for specifying reindexing script for system index migration [#119001](https://github.com/elastic/elasticsearch/pull/119001) * Bump major version for feature migration system indices [#117243](https://github.com/elastic/elasticsearch/pull/117243) * Change default Docker image to be based on UBI minimal instead of Ubuntu [#116739](https://github.com/elastic/elasticsearch/pull/116739) +* Give Kibana user 'all' permissions for .entity_analytics.* indices [#123588](https://github.com/elastic/elasticsearch/pull/123588) +* Improve size limiting string message [#122427](https://github.com/elastic/elasticsearch/pull/122427) * Infrastructure for assuming cluster features in the next major version [#118143](https://github.com/elastic/elasticsearch/pull/118143) * Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#124865](https://github.com/elastic/elasticsearch/pull/124865) +* Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#125117](https://github.com/elastic/elasticsearch/pull/125117) * Update ASM 9.7 -> 9.7.1 to support JDK 24 [#118094](https://github.com/elastic/elasticsearch/pull/118094) Infra/Metrics: @@ -214,21 +277,25 @@ Infra/Metrics: Infra/REST API: * A new query parameter `?include_source_on_error` was added for create / index, update and bulk REST APIs to control if to include the document source in the error response in case of parsing errors. The default value is `true`. [#120725](https://github.com/elastic/elasticsearch/pull/120725) +* Indicate when errors represent timeouts [#124936](https://github.com/elastic/elasticsearch/pull/124936) Infra/Scripting: * Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts [#114002](https://github.com/elastic/elasticsearch/pull/114002) Infra/Settings: +* Allow passing several reserved state chunks in single process call [#124574](https://github.com/elastic/elasticsearch/pull/124574) * Introduce `IndexSettingDeprecatedInV8AndRemovedInV9` Setting property [#120334](https://github.com/elastic/elasticsearch/pull/120334) * Run `TransportClusterGetSettingsAction` on local node [#119831](https://github.com/elastic/elasticsearch/pull/119831) Ingest Node: +* Allow setting the `type` in the reroute processor [#122409](https://github.com/elastic/elasticsearch/pull/122409) (issue: [#122409](https://github.com/elastic/elasticsearch/pull/122409)) * Optimize `IngestCtxMap` construction [#120833](https://github.com/elastic/elasticsearch/pull/120833) * Optimize `IngestDocMetadata` `isAvailable` [#120753](https://github.com/elastic/elasticsearch/pull/120753) * Optimize `IngestDocument` `FieldPath` allocation [#120573](https://github.com/elastic/elasticsearch/pull/120573) * Optimize some per-document hot paths in the geoip processor [#120824](https://github.com/elastic/elasticsearch/pull/120824) * Returning ignored fields in the simulate ingest API [#117214](https://github.com/elastic/elasticsearch/pull/117214) * Run `GetPipelineTransportAction` on local node [#120445](https://github.com/elastic/elasticsearch/pull/120445) +* Run `TransportEnrichStatsAction` on local node [#121256](https://github.com/elastic/elasticsearch/pull/121256) * Run `TransportGetEnrichPolicyAction` on local node [#121124](https://github.com/elastic/elasticsearch/pull/121124) * Run template simulation actions on local node [#120038](https://github.com/elastic/elasticsearch/pull/120038) @@ -245,28 +312,56 @@ Machine Learning: * Add DeBERTa-V2/V3 tokenizer [#111852](https://github.com/elastic/elasticsearch/pull/111852) * Add Inference Unified API for chat completions for OpenAI [#117589](https://github.com/elastic/elasticsearch/pull/117589) * Add Jina AI API to do inference for Embedding and Rerank models [#118652](https://github.com/elastic/elasticsearch/pull/118652) +* Add `ModelRegistryMetadata` to Cluster State [#121106](https://github.com/elastic/elasticsearch/pull/121106) * Add enterprise license check for Inference API actions [#119893](https://github.com/elastic/elasticsearch/pull/119893) * Adding chunking settings to `IbmWatsonxService` [#114914](https://github.com/elastic/elasticsearch/pull/114914) +* Adding common rerank options to Perform Inference API [#125239](https://github.com/elastic/elasticsearch/pull/125239) (issue: [#125239](https://github.com/elastic/elasticsearch/pull/125239)) * Adding default endpoint for Elastic Rerank [#117939](https://github.com/elastic/elasticsearch/pull/117939) +* Adding elser default endpoint for EIS [#122066](https://github.com/elastic/elasticsearch/pull/122066) * Adding endpoint creation validation for all task types to remaining services [#115020](https://github.com/elastic/elasticsearch/pull/115020) +* Adding endpoint creation validation to `ElasticInferenceService` [#117642](https://github.com/elastic/elasticsearch/pull/117642) +* Adding integration for VoyageAI embeddings and rerank models [#122134](https://github.com/elastic/elasticsearch/pull/122134) +* Adding support for binary embedding type to Cohere service embedding type [#120751](https://github.com/elastic/elasticsearch/pull/120751) +* Adding support for specifying embedding type to Jina AI service settings [#121548](https://github.com/elastic/elasticsearch/pull/121548) * Automatically rollover legacy .ml-anomalies indices [#120913](https://github.com/elastic/elasticsearch/pull/120913) * Automatically rollover legacy ml indices [#120405](https://github.com/elastic/elasticsearch/pull/120405) * Change the auditor to write via an alias [#120064](https://github.com/elastic/elasticsearch/pull/120064) * Check for presence of error object when validating streaming responses from integrations in the inference API [#118375](https://github.com/elastic/elasticsearch/pull/118375) +* Check if the anomaly results index has been rolled over [#125404](https://github.com/elastic/elasticsearch/pull/125404) +* ES|QL `change_point` processing command [#120998](https://github.com/elastic/elasticsearch/pull/120998) * ES|QL categorize with multiple groupings [#118173](https://github.com/elastic/elasticsearch/pull/118173) +* Expose `input_type` option at root level for `text_embedding` task type in Perform Inference API [#122638](https://github.com/elastic/elasticsearch/pull/122638) (issue: [#122638](https://github.com/elastic/elasticsearch/pull/122638)) * Ignore failures from renormalizing buckets in read-only index [#118674](https://github.com/elastic/elasticsearch/pull/118674) * Inference duration and error metrics [#115876](https://github.com/elastic/elasticsearch/pull/115876) +* Integrate with `DeepSeek` API [#122218](https://github.com/elastic/elasticsearch/pull/122218) +* Limit the number of chunks for semantic text to prevent high memory usage [#123150](https://github.com/elastic/elasticsearch/pull/123150) * Migrate stream to core error parsing [#120722](https://github.com/elastic/elasticsearch/pull/120722) * Remove all mentions of eis and gateway and deprecate flags that do [#116692](https://github.com/elastic/elasticsearch/pull/116692) * Remove deprecated sort from reindex operation within dataframe analytics procedure [#117606](https://github.com/elastic/elasticsearch/pull/117606) * Retry on `ClusterBlockException` on transform destination index [#118194](https://github.com/elastic/elasticsearch/pull/118194) * Support mTLS for the Elastic Inference Service integration inside the inference API [#119679](https://github.com/elastic/elasticsearch/pull/119679) +* Upgrade AWS v2 SDK to 2.30.38 [#124738](https://github.com/elastic/elasticsearch/pull/124738) * [Inference API] Add node-local rate limiting for the inference API [#120400](https://github.com/elastic/elasticsearch/pull/120400) +* [Inference API] Propagate product use case http header to EIS [#124025](https://github.com/elastic/elasticsearch/pull/124025) * [Inference API] fix spell words: covertToString to convertToString [#119922](https://github.com/elastic/elasticsearch/pull/119922) Mapping: * Add Optional Source Filtering to Source Loaders [#113827](https://github.com/elastic/elasticsearch/pull/113827) * Add option to store `sparse_vector` outside `_source` [#117917](https://github.com/elastic/elasticsearch/pull/117917) +* Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. [#122615](https://github.com/elastic/elasticsearch/pull/122615) (issue: [#122615](https://github.com/elastic/elasticsearch/pull/122615)) +* Enable the use of nested field type with index.mode=time_series [#122224](https://github.com/elastic/elasticsearch/pull/122224) (issue: [#122224](https://github.com/elastic/elasticsearch/pull/122224)) +* Improved error message when index field type is invalid [#122860](https://github.com/elastic/elasticsearch/pull/122860) +* Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields [#119546](https://github.com/elastic/elasticsearch/pull/119546) +* Release semantic_text as a GA feature [#124669](https://github.com/elastic/elasticsearch/pull/124669) +* Store arrays offsets for boolean fields natively with synthetic source [#125529](https://github.com/elastic/elasticsearch/pull/125529) +* Store arrays offsets for ip fields natively with synthetic source [#122999](https://github.com/elastic/elasticsearch/pull/122999) +* Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. [#113757](https://github.com/elastic/elasticsearch/pull/113757) +* Store arrays offsets for numeric fields natively with synthetic source [#124594](https://github.com/elastic/elasticsearch/pull/124594) +* Store arrays offsets for unsigned long fields natively with synthetic source [#125709](https://github.com/elastic/elasticsearch/pull/125709) +* Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` [#124927](https://github.com/elastic/elasticsearch/pull/124927) +* Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields [#122637](https://github.com/elastic/elasticsearch/pull/122637) +* Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields [#124050](https://github.com/elastic/elasticsearch/pull/124050) +* Use `FallbackSyntheticSourceBlockLoader` for number fields [#122280](https://github.com/elastic/elasticsearch/pull/122280) Network: * Allow http unsafe buffers by default [#116115](https://github.com/elastic/elasticsearch/pull/116115) @@ -274,8 +369,12 @@ Network: * Remove HTTP content copies [#117303](https://github.com/elastic/elasticsearch/pull/117303) * `ConnectTransportException` returns retryable BAD_GATEWAY [#118681](https://github.com/elastic/elasticsearch/pull/118681) (issue: [#118681](https://github.com/elastic/elasticsearch/pull/118681)) +Packaging: +* Update bundled JDK to Java 24 [#125159](https://github.com/elastic/elasticsearch/pull/125159) + Ranking: * Add a generic `rescorer` retriever based on the search request's rescore functionality [#118585](https://github.com/elastic/elasticsearch/pull/118585) (issue: [#118585](https://github.com/elastic/elasticsearch/pull/118585)) +* Leverage scorer supplier in `QueryFeatureExtractor` [#125259](https://github.com/elastic/elasticsearch/pull/125259) * Set default reranker for text similarity reranker to Elastic reranker [#120551](https://github.com/elastic/elasticsearch/pull/120551) Recovery: @@ -287,17 +386,22 @@ Reindex: Relevance: * Add Multi-Field Support for Semantic Text Fields [#120128](https://github.com/elastic/elasticsearch/pull/120128) +* Skip semantic_text embedding generation when no content is provided. [#123763](https://github.com/elastic/elasticsearch/pull/123763) Search: +* Account for the `SearchHit` source in circuit breaker [#121920](https://github.com/elastic/elasticsearch/pull/121920) (issue: [#121920](https://github.com/elastic/elasticsearch/pull/121920)) * Add match support for `semantic_text` fields [#117839](https://github.com/elastic/elasticsearch/pull/117839) * Add support for `sparse_vector` queries against `semantic_text` fields [#118617](https://github.com/elastic/elasticsearch/pull/118617) * Add support for knn vector queries on `semantic_text` fields [#119011](https://github.com/elastic/elasticsearch/pull/119011) +* Added optional parameters to QSTR ES|QL function [#121787](https://github.com/elastic/elasticsearch/pull/121787) (issue: [#121787](https://github.com/elastic/elasticsearch/pull/121787)) * Adding linear retriever to support weighted sums of sub-retrievers [#120222](https://github.com/elastic/elasticsearch/pull/120222) * Address and remove any references of RestApiVersion version 7 [#117572](https://github.com/elastic/elasticsearch/pull/117572) * Feat: add a user-configurable timeout parameter to the `_resolve/cluster` API [#120542](https://github.com/elastic/elasticsearch/pull/120542) * Introduce batched query execution and data-node side reduce [#121885](https://github.com/elastic/elasticsearch/pull/121885) * Make semantic text part of the text family [#119792](https://github.com/elastic/elasticsearch/pull/119792) * Only aggregations require at least one shard request [#115314](https://github.com/elastic/elasticsearch/pull/115314) +* Optimize memory usage in `ShardBulkInferenceActionFilter` [#124313](https://github.com/elastic/elasticsearch/pull/124313) +* Optionally allow text similarity reranking to fail [#121784](https://github.com/elastic/elasticsearch/pull/121784) * Prevent data nodes from sending stack traces to coordinator when `error_trace=false` [#118266](https://github.com/elastic/elasticsearch/pull/118266) * Propagate status codes from shard failures appropriately [#118016](https://github.com/elastic/elasticsearch/pull/118016) (issue: [#118016](https://github.com/elastic/elasticsearch/pull/118016)) * Upgrade to Lucene 10 [#114741](https://github.com/elastic/elasticsearch/pull/114741) @@ -309,9 +413,18 @@ Security: Snapshot/Restore: * Add IMDSv2 support to `repository-s3` [#117748](https://github.com/elastic/elasticsearch/pull/117748) (issue: [#117748](https://github.com/elastic/elasticsearch/pull/117748)) * Expose operation and request counts separately in repository stats [#117530](https://github.com/elastic/elasticsearch/pull/117530) (issue: [#117530](https://github.com/elastic/elasticsearch/pull/117530)) +* GCS blob store: add `OperationPurpose/Operation` stats counters [#122991](https://github.com/elastic/elasticsearch/pull/122991) * Retry `S3BlobContainer#getRegister` on all exceptions [#114813](https://github.com/elastic/elasticsearch/pull/114813) * Retry internally when CAS upload is throttled [GCS] [#120250](https://github.com/elastic/elasticsearch/pull/120250) (issue: [#120250](https://github.com/elastic/elasticsearch/pull/120250)) +* Retry when the server can't be resolved (Google Cloud Storage) [#123852](https://github.com/elastic/elasticsearch/pull/123852) * Track shard snapshot progress during node shutdown [#112567](https://github.com/elastic/elasticsearch/pull/112567) +* Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) + +Stats: +* Run XPack usage actions on local node [#122933](https://github.com/elastic/elasticsearch/pull/122933) + +Store: +* Abort pending deletion on `IndicesService` close [#123569](https://github.com/elastic/elasticsearch/pull/123569) Suggesters: * Extensible Completion Postings Formats [#111494](https://github.com/elastic/elasticsearch/pull/111494) @@ -327,19 +440,33 @@ Transform: * [Deprecation] Add `transform_ids` to outdated index [#120821](https://github.com/elastic/elasticsearch/pull/120821) Vector Search: +* Add bit vector support to semantic text [#123187](https://github.com/elastic/elasticsearch/pull/123187) * Add new experimental `rank_vectors` mapping for late-interaction second order ranking [#118804](https://github.com/elastic/elasticsearch/pull/118804) +* Add panama implementations of byte-bit and float-bit script operations [#124722](https://github.com/elastic/elasticsearch/pull/124722) (issue: [#124722](https://github.com/elastic/elasticsearch/pull/124722)) +* Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields [#122381](https://github.com/elastic/elasticsearch/pull/122381) (issue: [#122381](https://github.com/elastic/elasticsearch/pull/122381)) +* Allow zero for `rescore_vector.oversample` to indicate by-passing oversample and rescoring [#125599](https://github.com/elastic/elasticsearch/pull/125599) * Even better(er) binary quantization [#117994](https://github.com/elastic/elasticsearch/pull/117994) * KNN vector rescoring for quantized vectors [#116663](https://github.com/elastic/elasticsearch/pull/116663) * Mark bbq indices as GA and add rolling upgrade integration tests [#121105](https://github.com/elastic/elasticsearch/pull/121105) +* New `vector_rescore` parameter as a quantized index type option [#124581](https://github.com/elastic/elasticsearch/pull/124581) * Speed up bit compared with floats or bytes script operations [#117199](https://github.com/elastic/elasticsearch/pull/117199) +Watcher: +* Run `TransportGetWatcherSettingsAction` on local node [#122857](https://github.com/elastic/elasticsearch/pull/122857) + ### Fixes [elasticsearch-900-fixes] Aggregations: +* Aggs: Let terms run in global ords mode no match [#124782](https://github.com/elastic/elasticsearch/pull/124782) * Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations [#116174](https://github.com/elastic/elasticsearch/pull/116174) (issue: [#116174](https://github.com/elastic/elasticsearch/pull/116174)) +Allocation: +* `DesiredBalanceReconciler` always returns `AllocationStats` [#122458](https://github.com/elastic/elasticsearch/pull/122458) + Analysis: +* Adjust exception thrown when unable to load hunspell dict [#123743](https://github.com/elastic/elasticsearch/pull/123743) * Analyze API to return 400 for wrong custom analyzer [#121568](https://github.com/elastic/elasticsearch/pull/121568) (issue: [#121568](https://github.com/elastic/elasticsearch/pull/121568)) +* Non existing synonyms sets do not fail shard recovery for indices [#125659](https://github.com/elastic/elasticsearch/pull/125659) (issue: [#125659](https://github.com/elastic/elasticsearch/pull/125659)) Authentication: * Fix NPE for missing Content Type header in OIDC Authenticator [#126191](https://github.com/elastic/elasticsearch/pull/126191) @@ -351,6 +478,9 @@ CRUD: * Preserve thread context when waiting for segment generation in RTG [#114623](https://github.com/elastic/elasticsearch/pull/114623) * Preserve thread context when waiting for segment generation in RTG [#117148](https://github.com/elastic/elasticsearch/pull/117148) +Cluster Coordination: +* Disable logging in `ClusterFormationFailureHelper` on shutdown [#125244](https://github.com/elastic/elasticsearch/pull/125244) (issue: [#125244](https://github.com/elastic/elasticsearch/pull/125244)) + Data streams: * Avoid updating settings version in `MetadataMigrateToDataStreamService` when settings have not changed [#118704](https://github.com/elastic/elasticsearch/pull/118704) * Block-writes cannot be added after read-only [#119007](https://github.com/elastic/elasticsearch/pull/119007) (issue: [#119007](https://github.com/elastic/elasticsearch/pull/119007)) @@ -358,15 +488,32 @@ Data streams: * Fixes a invalid warning from being issued when restoring a system data stream from a snapshot. [#125881](https://github.com/elastic/elasticsearch/pull/125881) * Match dot prefix of migrated DS backing index with the source index [#120042](https://github.com/elastic/elasticsearch/pull/120042) * Refresh source index before reindexing data stream index [#120752](https://github.com/elastic/elasticsearch/pull/120752) (issue: [#120752](https://github.com/elastic/elasticsearch/pull/120752)) +* Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover [#122905](https://github.com/elastic/elasticsearch/pull/122905) * `ReindexDataStreamIndex` bug in assertion caused by reference equality [#121325](https://github.com/elastic/elasticsearch/pull/121325) +Distributed: +* Pass `IndexReshardingMetadata` over the wire [#124841](https://github.com/elastic/elasticsearch/pull/124841) + Downsampling: * Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` [#121727](https://github.com/elastic/elasticsearch/pull/121727) (issues: [#121727](https://github.com/elastic/elasticsearch/pull/121727), [#121727](https://github.com/elastic/elasticsearch/pull/121727)) +* Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. [#124451](https://github.com/elastic/elasticsearch/pull/124451) + +EQL: +* Fix EQL double invoking listener [#124918](https://github.com/elastic/elasticsearch/pull/124918) ES|QL: +* Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: [#122886](https://github.com/elastic/elasticsearch/pull/122886)) * Allow the data type of `null` in filters [#118324](https://github.com/elastic/elasticsearch/pull/118324) (issue: [#118324](https://github.com/elastic/elasticsearch/pull/118324)) +* Avoid over collecting in Limit or Lucene Operator [#123296](https://github.com/elastic/elasticsearch/pull/123296) +* Change the order of the optimization rules [#124335](https://github.com/elastic/elasticsearch/pull/124335) * Correct line and column numbers of missing named parameters [#120852](https://github.com/elastic/elasticsearch/pull/120852) * Drop null columns in text formats [#117643](https://github.com/elastic/elasticsearch/pull/117643) (issue: [#117643](https://github.com/elastic/elasticsearch/pull/117643)) +* ESQL - date nanos range bug? [#125345](https://github.com/elastic/elasticsearch/pull/125345) (issue: [#125345](https://github.com/elastic/elasticsearch/pull/125345)) +* ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate` [#124446](https://github.com/elastic/elasticsearch/pull/124446) (issue: [#124446](https://github.com/elastic/elasticsearch/pull/124446)) +* ESQL: Fix inconsistent results in using scaled_float field [#122586](https://github.com/elastic/elasticsearch/pull/122586) (issue: [#122586](https://github.com/elastic/elasticsearch/pull/122586)) +* ESQL: Remove estimated row size assertion [#122762](https://github.com/elastic/elasticsearch/pull/122762) (issue: [#122762](https://github.com/elastic/elasticsearch/pull/122762)) +* ES|QL: Fix scoring for full text functions [#124540](https://github.com/elastic/elasticsearch/pull/124540) +* Esql - Fix lucene push down behavior when a range contains nanos and millis [#125595](https://github.com/elastic/elasticsearch/pull/125595) * Fix ROUND() with unsigned longs throwing in some edge cases [#119536](https://github.com/elastic/elasticsearch/pull/119536) * Fix TDigestState.read CB leaks [#114303](https://github.com/elastic/elasticsearch/pull/114303) (issue: [#114303](https://github.com/elastic/elasticsearch/pull/114303)) * Fix TopN row size estimate [#119476](https://github.com/elastic/elasticsearch/pull/119476) (issue: [#119476](https://github.com/elastic/elasticsearch/pull/119476)) @@ -376,27 +523,52 @@ ES|QL: * Fix async stop sometimes not properly collecting result [#121843](https://github.com/elastic/elasticsearch/pull/121843) (issue: [#121843](https://github.com/elastic/elasticsearch/pull/121843)) * Fix attribute set equals [#118823](https://github.com/elastic/elasticsearch/pull/118823) * Fix double lookup failure on ESQL [#115616](https://github.com/elastic/elasticsearch/pull/115616) (issue: [#115616](https://github.com/elastic/elasticsearch/pull/115616)) +* Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: [#123492](https://github.com/elastic/elasticsearch/pull/123492)) +* Fix functions emitting warnings with no source [#122821](https://github.com/elastic/elasticsearch/pull/122821) (issue: [#122821](https://github.com/elastic/elasticsearch/pull/122821)) * Fix queries with document level security on lookup indexes [#120617](https://github.com/elastic/elasticsearch/pull/120617) (issue: [#120617](https://github.com/elastic/elasticsearch/pull/120617)) * Fix writing for LOOKUP status [#119296](https://github.com/elastic/elasticsearch/pull/119296) (issue: [#119296](https://github.com/elastic/elasticsearch/pull/119296)) +* Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: [#122601](https://github.com/elastic/elasticsearch/pull/122601)) +* Improve error message for ( and [ [#124177](https://github.com/elastic/elasticsearch/pull/124177) (issue: [#124177](https://github.com/elastic/elasticsearch/pull/124177)) +* Lazy collection copying during node transform [#124424](https://github.com/elastic/elasticsearch/pull/124424) * Limit memory usage of `fold` [#118602](https://github.com/elastic/elasticsearch/pull/118602) * Limit size of query [#117898](https://github.com/elastic/elasticsearch/pull/117898) +* Make `numberOfChannels` consistent with layout map by removing duplicated `ChannelSet` [#125636](https://github.com/elastic/elasticsearch/pull/125636) +* Reduce iteration complexity for plan traversal [#123427](https://github.com/elastic/elasticsearch/pull/123427) +* Remove duplicated nested commands [#123085](https://github.com/elastic/elasticsearch/pull/123085) * Remove redundant sorts from execution plan [#121156](https://github.com/elastic/elasticsearch/pull/121156) * Revert unwanted ES|QL lexer changes from PR #120354 [#120538](https://github.com/elastic/elasticsearch/pull/120538) +* Revive inlinestats [#122257](https://github.com/elastic/elasticsearch/pull/122257) +* Revive some more of inlinestats functionality [#123589](https://github.com/elastic/elasticsearch/pull/123589) +* TO_LOWER processes all values [#124676](https://github.com/elastic/elasticsearch/pull/124676) (issue: [#124676](https://github.com/elastic/elasticsearch/pull/124676)) +* Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: [#124001](https://github.com/elastic/elasticsearch/pull/124001)) + +Engine: +* Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: [#123010](https://github.com/elastic/elasticsearch/pull/123010)) Health: * Do not recommend increasing `max_shards_per_node` [#120458](https://github.com/elastic/elasticsearch/pull/120458) +Highlighting: +* Restore V8 REST compatibility around highlight `force_source` parameter [#124873](https://github.com/elastic/elasticsearch/pull/124873) + Indices APIs: * Add `?master_timeout` to `POST /_ilm/migrate_to_data_tiers` [#120883](https://github.com/elastic/elasticsearch/pull/120883) +* Fix NPE in rolling over unknown target and return 404 [#125352](https://github.com/elastic/elasticsearch/pull/125352) * Fix broken yaml test `30_create_from` [#120662](https://github.com/elastic/elasticsearch/pull/120662) * Include hidden indices in `DeprecationInfoAction` [#118035](https://github.com/elastic/elasticsearch/pull/118035) (issue: [#118035](https://github.com/elastic/elasticsearch/pull/118035)) * Preventing `ConcurrentModificationException` when updating settings for more than one index [#126077](https://github.com/elastic/elasticsearch/pull/126077) +* Updates the deprecation info API to not warn about system indices and data streams [#122951](https://github.com/elastic/elasticsearch/pull/122951) Inference: * [Inference API] Put back legacy EIS URL setting [#121207](https://github.com/elastic/elasticsearch/pull/121207) Infra/Core: * Epoch Millis Rounding Down and Not Up 2 [#118353](https://github.com/elastic/elasticsearch/pull/118353) +* Fix system data streams to be restorable from a snapshot [#124651](https://github.com/elastic/elasticsearch/pull/124651) (issue: [#124651](https://github.com/elastic/elasticsearch/pull/124651)) +* Have create index return a bad request on poor formatting [#123761](https://github.com/elastic/elasticsearch/pull/123761) +* Include data streams when converting an existing resource to a system resource [#121392](https://github.com/elastic/elasticsearch/pull/121392) +* Reduce Data Loss in System Indices Migration [#121327](https://github.com/elastic/elasticsearch/pull/121327) +* System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) * System data streams are not being upgraded in the feature migration API [#124884](https://github.com/elastic/elasticsearch/pull/124884) (issue: [#124884](https://github.com/elastic/elasticsearch/pull/124884)) * Wrap jackson exception on malformed json string [#114445](https://github.com/elastic/elasticsearch/pull/114445) (issue: [#114445](https://github.com/elastic/elasticsearch/pull/114445)) @@ -422,41 +594,73 @@ Infra/Settings: Ingest Node: * Add warning headers for ingest pipelines containing special characters [#114837](https://github.com/elastic/elasticsearch/pull/114837) (issue: [#114837](https://github.com/elastic/elasticsearch/pull/114837)) * Fix geoip databases index access after system feature migration [#121196](https://github.com/elastic/elasticsearch/pull/121196) +* Fix geoip databases index access after system feature migration (again) [#122938](https://github.com/elastic/elasticsearch/pull/122938) +* Fix geoip databases index access after system feature migration (take 3) [#124604](https://github.com/elastic/elasticsearch/pull/124604) +* apm-data: Use representative count as event.success_count if available [#119995](https://github.com/elastic/elasticsearch/pull/119995) Logs: * Always check if index mode is logsdb [#116922](https://github.com/elastic/elasticsearch/pull/116922) Machine Learning: +* Add `ElasticInferenceServiceCompletionServiceSettings` [#123155](https://github.com/elastic/elasticsearch/pull/123155) +* Add enterprise license check to inference action for semantic text fields [#122293](https://github.com/elastic/elasticsearch/pull/122293) +* Avoid potentially throwing calls to Task#getDescription in model download [#124527](https://github.com/elastic/elasticsearch/pull/124527) * Change format for Unified Chat [#121396](https://github.com/elastic/elasticsearch/pull/121396) +* Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` [#125023](https://github.com/elastic/elasticsearch/pull/125023) * Fix get all inference endponts not returning multiple endpoints sharing model deployment [#121821](https://github.com/elastic/elasticsearch/pull/121821) +* Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) * Fixing bedrock event executor terminated cache issue [#118177](https://github.com/elastic/elasticsearch/pull/118177) (issue: [#118177](https://github.com/elastic/elasticsearch/pull/118177)) * Fixing bug setting index when parsing Google Vertex AI results [#117287](https://github.com/elastic/elasticsearch/pull/117287) * Prevent get datafeeds stats API returning an error when local tasks are slow to stop [#125477](https://github.com/elastic/elasticsearch/pull/125477) (issue: [#125477](https://github.com/elastic/elasticsearch/pull/125477)) +* Provide model size statistics as soon as an anomaly detection job is opened [#124638](https://github.com/elastic/elasticsearch/pull/124638) (issue: [#124638](https://github.com/elastic/elasticsearch/pull/124638)) +* Retry on streaming errors [#123076](https://github.com/elastic/elasticsearch/pull/123076) +* Return a Conflict status code if the model deployment is stopped by a user [#125204](https://github.com/elastic/elasticsearch/pull/125204) (issue: [#125204](https://github.com/elastic/elasticsearch/pull/125204)) +* Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) +* Set default similarity for Cohere model to cosine [#125370](https://github.com/elastic/elasticsearch/pull/125370) (issue: [#125370](https://github.com/elastic/elasticsearch/pull/125370)) +* Updates to allow using Cohere binary embedding response in semantic search queries [#121827](https://github.com/elastic/elasticsearch/pull/121827) * Updating Inference Update API documentation to have the correct PUT method [#121048](https://github.com/elastic/elasticsearch/pull/121048) * Wait for up to 2 seconds for yellow status before starting search [#115938](https://github.com/elastic/elasticsearch/pull/115938) (issues: [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938)) +* [Inference API] Fix output stream ordering in `InferenceActionProxy` [#124225](https://github.com/elastic/elasticsearch/pull/124225) * [Inference API] Fix unique ID message for inference ID matches trained model ID [#119543](https://github.com/elastic/elasticsearch/pull/119543) (issue: [#119543](https://github.com/elastic/elasticsearch/pull/119543)) Mapping: +* Avoid serializing empty `_source` fields in mappings [#122606](https://github.com/elastic/elasticsearch/pull/122606) * Enable New Semantic Text Format Only On Newly Created Indices [#121556](https://github.com/elastic/elasticsearch/pull/121556) +* Fix Semantic Text 8.x Upgrade Bug [#125446](https://github.com/elastic/elasticsearch/pull/125446) * Fix propagation of dynamic mapping parameter when applying `copy_to` [#121109](https://github.com/elastic/elasticsearch/pull/121109) (issue: [#121109](https://github.com/elastic/elasticsearch/pull/121109)) * Fix realtime get of nested fields with synthetic source [#119575](https://github.com/elastic/elasticsearch/pull/119575) (issue: [#119575](https://github.com/elastic/elasticsearch/pull/119575)) * Merge field mappers when updating mappings with [subobjects:false] [#120370](https://github.com/elastic/elasticsearch/pull/120370) (issue: [#120370](https://github.com/elastic/elasticsearch/pull/120370)) +* Merge template mappings properly during validation [#124784](https://github.com/elastic/elasticsearch/pull/124784) (issue: [#124784](https://github.com/elastic/elasticsearch/pull/124784)) * Tweak `copy_to` handling in synthetic `_source` to account for nested objects [#120974](https://github.com/elastic/elasticsearch/pull/120974) (issue: [#120974](https://github.com/elastic/elasticsearch/pull/120974)) Network: * Remove ChunkedToXContentBuilder [#119310](https://github.com/elastic/elasticsearch/pull/119310) (issue: [#119310](https://github.com/elastic/elasticsearch/pull/119310)) +Ranking: +* Fix LTR query feature with phrases (and two-phase) queries [#125103](https://github.com/elastic/elasticsearch/pull/125103) +* Restore `TextSimilarityRankBuilder` XContent output [#124564](https://github.com/elastic/elasticsearch/pull/124564) + +Relevance: +* Prevent Query Rule Creation with Invalid Numeric Match Criteria [#122823](https://github.com/elastic/elasticsearch/pull/122823) + Search: * Catch and handle disconnect exceptions in search [#115836](https://github.com/elastic/elasticsearch/pull/115836) +* Fix handling of auto expand replicas for stateless indices [#122365](https://github.com/elastic/elasticsearch/pull/122365) * Fix leak in `DfsQueryPhase` and introduce search disconnect stress test [#116060](https://github.com/elastic/elasticsearch/pull/116060) (issue: [#116060](https://github.com/elastic/elasticsearch/pull/116060)) * Fix/QueryBuilderBWCIT_muted_test [#117831](https://github.com/elastic/elasticsearch/pull/117831) +* Handle long overflow in dates [#124048](https://github.com/elastic/elasticsearch/pull/124048) (issue: [#124048](https://github.com/elastic/elasticsearch/pull/124048)) +* Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: [#122357](https://github.com/elastic/elasticsearch/pull/122357)) * In this pr, a 400 error is returned when _source / _seq_no / _feature / _nested_path / _field_names is requested, rather a 5xx [#117229](https://github.com/elastic/elasticsearch/pull/117229) * Inconsistency in the _analyzer api when the index is not included [#115930](https://github.com/elastic/elasticsearch/pull/115930) +* Let MLTQuery throw IAE when no analyzer is set [#124662](https://github.com/elastic/elasticsearch/pull/124662) (issue: [#124662](https://github.com/elastic/elasticsearch/pull/124662)) +* Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` [#125650](https://github.com/elastic/elasticsearch/pull/125650) (issue: [#125650](https://github.com/elastic/elasticsearch/pull/125650)) * Log stack traces on data nodes before they are cleared for transport [#125732](https://github.com/elastic/elasticsearch/pull/125732) +* Minor-Fixes Support 7x segments as archive in 8x / 9x [#125666](https://github.com/elastic/elasticsearch/pull/125666) * Re-enable parallel collection for field sorted top hits [#125916](https://github.com/elastic/elasticsearch/pull/125916) * Remove duplicate code in ESIntegTestCase [#120799](https://github.com/elastic/elasticsearch/pull/120799) * SearchStatesIt failures reported by CI [#117618](https://github.com/elastic/elasticsearch/pull/117618) (issues: [#117618](https://github.com/elastic/elasticsearch/pull/117618), [#117618](https://github.com/elastic/elasticsearch/pull/117618)) * Skip fetching _inference_fields field in legacy semantic_text format [#121720](https://github.com/elastic/elasticsearch/pull/121720) +* Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. [#119503](https://github.com/elastic/elasticsearch/pull/119503) (issue: [#119503](https://github.com/elastic/elasticsearch/pull/119503)) * Test/107515 restore template with match only text mapper it fail [#120392](https://github.com/elastic/elasticsearch/pull/120392) (issue: [#120392](https://github.com/elastic/elasticsearch/pull/120392)) * Updated Date Range to Follow Documentation When Assuming Missing Values [#112258](https://github.com/elastic/elasticsearch/pull/112258) (issue: [#112258](https://github.com/elastic/elasticsearch/pull/112258)) * `CrossClusterIT` `testCancel` failure [#117750](https://github.com/elastic/elasticsearch/pull/117750) (issue: [#117750](https://github.com/elastic/elasticsearch/pull/117750)) @@ -464,315 +668,15 @@ Search: Snapshot/Restore: * Add undeclared Azure settings, modify test to exercise them [#118634](https://github.com/elastic/elasticsearch/pull/118634) +* Fork post-snapshot-delete cleanup off master thread [#122731](https://github.com/elastic/elasticsearch/pull/122731) +* Limit number of suppressed S3 deletion errors [#123630](https://github.com/elastic/elasticsearch/pull/123630) (issue: [#123630](https://github.com/elastic/elasticsearch/pull/123630)) * Retry throttled snapshot deletions [#113237](https://github.com/elastic/elasticsearch/pull/113237) +* This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. [#124931](https://github.com/elastic/elasticsearch/pull/124931) * Use the system index descriptor in the snapshot blob cache cleanup task [#120937](https://github.com/elastic/elasticsearch/pull/120937) (issue: [#120937](https://github.com/elastic/elasticsearch/pull/120937)) Store: * Do not capture `ClusterChangedEvent` in `IndicesStore` call to #onClusterStateShardsClosed [#120193](https://github.com/elastic/elasticsearch/pull/120193) -Vector Search: -* Apply default k for knn query eagerly [#118774](https://github.com/elastic/elasticsearch/pull/118774) -* Fix `bbq_hnsw` merge file cleanup on random IO exceptions [#119691](https://github.com/elastic/elasticsearch/pull/119691) (issue: [#119691](https://github.com/elastic/elasticsearch/pull/119691)) - -Watcher: -* Watcher history index has too many indexed fields - [#117701](https://github.com/elastic/elasticsearch/pull/117701) (issue: [#117701](https://github.com/elastic/elasticsearch/pull/117701)) - - -## 9.1.0 [elasticsearch-910-release-notes] - -### Highlights [elasticsearch-910-highlights] - -::::{dropdown} Release semantic_text as a GA feature -semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. - -For more information, check [PR #124669](https://github.com/elastic/elasticsearch/pull/124669). -:::: - -### Features and enhancements [elasticsearch-910-features-enhancements] - -Allocation: -* Add cache support in `TransportGetAllocationStatsAction` [#124898](https://github.com/elastic/elasticsearch/pull/124898) (issue: [#124898](https://github.com/elastic/elasticsearch/pull/124898)) -* Introduce `AllocationBalancingRoundSummaryService` [#120957](https://github.com/elastic/elasticsearch/pull/120957) - -Authorization: -* Do not fetch reserved roles from native store when Get Role API is called [#121971](https://github.com/elastic/elasticsearch/pull/121971) - -CRUD: -* Enhance memory accounting for document expansion and introduce max document size limit [#123543](https://github.com/elastic/elasticsearch/pull/123543) - -Data streams: -* Add index mode to get data stream API [#122486](https://github.com/elastic/elasticsearch/pull/122486) -* Reindex data stream indices on different nodes [#125171](https://github.com/elastic/elasticsearch/pull/125171) -* Retry ILM async action after reindexing data stream [#124149](https://github.com/elastic/elasticsearch/pull/124149) -* Run `TransportGetDataStreamLifecycleAction` on local node [#125214](https://github.com/elastic/elasticsearch/pull/125214) -* Run `TransportGetDataStreamOptionsAction` on local node [#125213](https://github.com/elastic/elasticsearch/pull/125213) -* Run `TransportGetDataStreamsAction` on local node [#122852](https://github.com/elastic/elasticsearch/pull/122852) -* Set cause on create index request in create from action [#124363](https://github.com/elastic/elasticsearch/pull/124363) - -Downsampling: -* Improve downsample performance by buffering docids and do bulk processing [#124477](https://github.com/elastic/elasticsearch/pull/124477) -* Improve rolling up metrics [#124739](https://github.com/elastic/elasticsearch/pull/124739) - -ES|QL: -* Add initial grammar and changes for FORK [#121948](https://github.com/elastic/elasticsearch/pull/121948) -* Add initial grammar and planning for RRF (snapshot) [#123396](https://github.com/elastic/elasticsearch/pull/123396) -* Allow partial results in ES|QL [#121942](https://github.com/elastic/elasticsearch/pull/121942) -* Allow skip shards with `_tier` and `_index` in ES|QL [#123728](https://github.com/elastic/elasticsearch/pull/123728) -* Avoid `NamedWritable` in block serialization [#124394](https://github.com/elastic/elasticsearch/pull/124394) -* Calculate concurrent node limit [#124901](https://github.com/elastic/elasticsearch/pull/124901) -* Double parameter markers for identifiers [#122459](https://github.com/elastic/elasticsearch/pull/122459) -* ESQL: Align `RENAME` behavior with `EVAL` for sequential processing [#122250](https://github.com/elastic/elasticsearch/pull/122250) (issue: [#122250](https://github.com/elastic/elasticsearch/pull/122250)) -* ES|QL - Add scoring for full text functions disjunctions [#121793](https://github.com/elastic/elasticsearch/pull/121793) -* ES|QL slow log [#124094](https://github.com/elastic/elasticsearch/pull/124094) -* ES|QL: Support `::date` in inline cast [#123460](https://github.com/elastic/elasticsearch/pull/123460) (issue: [#123460](https://github.com/elastic/elasticsearch/pull/123460)) -* Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: [#123290](https://github.com/elastic/elasticsearch/pull/123290)) -* Implement runtime skip_unavailable=true [#121240](https://github.com/elastic/elasticsearch/pull/121240) -* Include failures in partial response [#124929](https://github.com/elastic/elasticsearch/pull/124929) -* Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) -* Introduce `allow_partial_results` setting in ES|QL [#122890](https://github.com/elastic/elasticsearch/pull/122890) -* Introduce a pre-mapping logical plan processing step [#121260](https://github.com/elastic/elasticsearch/pull/121260) -* Keep ordinals in conversion functions [#125357](https://github.com/elastic/elasticsearch/pull/125357) -* Pragma to load from stored fields [#122891](https://github.com/elastic/elasticsearch/pull/122891) -* Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: [#123381](https://github.com/elastic/elasticsearch/pull/123381)) -* Remove page alignment in exchange sink [#124610](https://github.com/elastic/elasticsearch/pull/124610) -* Render `aggregate_metric_double` [#122660](https://github.com/elastic/elasticsearch/pull/122660) -* Report `original_types` [#124913](https://github.com/elastic/elasticsearch/pull/124913) -* Report failures on partial results [#124823](https://github.com/elastic/elasticsearch/pull/124823) -* Retry ES|QL node requests on shard level failures [#120774](https://github.com/elastic/elasticsearch/pull/120774) -* Reuse child `outputSet` inside the plan where possible [#124611](https://github.com/elastic/elasticsearch/pull/124611) -* Support partial results in CCS in ES|QL [#122708](https://github.com/elastic/elasticsearch/pull/122708) -* Support subset of metrics in aggregate metric double [#121805](https://github.com/elastic/elasticsearch/pull/121805) -* `ToAggregateMetricDouble` function [#124595](https://github.com/elastic/elasticsearch/pull/124595) - -Engine: -* Threadpool merge scheduler [#120869](https://github.com/elastic/elasticsearch/pull/120869) - -Health: -* Add health indicator impact to `HealthPeriodicLogger` [#122390](https://github.com/elastic/elasticsearch/pull/122390) - -ILM+SLM: -* Improve SLM Health Indicator to cover missing snapshot [#121370](https://github.com/elastic/elasticsearch/pull/121370) -* Process ILM cluster state updates on another thread [#123712](https://github.com/elastic/elasticsearch/pull/123712) -* Run `TransportExplainLifecycleAction` on local node [#122885](https://github.com/elastic/elasticsearch/pull/122885) -* Truncate `step_info` and error reason in ILM execution state and history [#125054](https://github.com/elastic/elasticsearch/pull/125054) (issue: [#125054](https://github.com/elastic/elasticsearch/pull/125054)) - -Indices APIs: -* Avoid creating known_fields for every check in Alias [#124690](https://github.com/elastic/elasticsearch/pull/124690) -* Run `TransportGetMappingsAction` on local node [#122921](https://github.com/elastic/elasticsearch/pull/122921) - -Inference: -* [Inference API] Rename `model_id` prop to model in EIS sparse inference request body [#122272](https://github.com/elastic/elasticsearch/pull/122272) - -Infra/CLI: -* Ignore _JAVA_OPTIONS [#124843](https://github.com/elastic/elasticsearch/pull/124843) - -Infra/Core: -* Give Kibana user 'all' permissions for .entity_analytics.* indices [#123588](https://github.com/elastic/elasticsearch/pull/123588) -* Improve size limiting string message [#122427](https://github.com/elastic/elasticsearch/pull/122427) -* Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#125117](https://github.com/elastic/elasticsearch/pull/125117) - -Infra/REST API: -* Indicate when errors represent timeouts [#124936](https://github.com/elastic/elasticsearch/pull/124936) - -Infra/Settings: -* Allow passing several reserved state chunks in single process call [#124574](https://github.com/elastic/elasticsearch/pull/124574) - -Ingest Node: -* Allow setting the `type` in the reroute processor [#122409](https://github.com/elastic/elasticsearch/pull/122409) (issue: [#122409](https://github.com/elastic/elasticsearch/pull/122409)) -* Run `TransportEnrichStatsAction` on local node [#121256](https://github.com/elastic/elasticsearch/pull/121256) - -Machine Learning: -* Add `ModelRegistryMetadata` to Cluster State [#121106](https://github.com/elastic/elasticsearch/pull/121106) -* Adding common rerank options to Perform Inference API [#125239](https://github.com/elastic/elasticsearch/pull/125239) (issue: [#125239](https://github.com/elastic/elasticsearch/pull/125239)) -* Adding elser default endpoint for EIS [#122066](https://github.com/elastic/elasticsearch/pull/122066) -* Adding endpoint creation validation to `ElasticInferenceService` [#117642](https://github.com/elastic/elasticsearch/pull/117642) -* Adding integration for VoyageAI embeddings and rerank models [#122134](https://github.com/elastic/elasticsearch/pull/122134) -* Adding support for binary embedding type to Cohere service embedding type [#120751](https://github.com/elastic/elasticsearch/pull/120751) -* Adding support for specifying embedding type to Jina AI service settings [#121548](https://github.com/elastic/elasticsearch/pull/121548) -* Check if the anomaly results index has been rolled over [#125404](https://github.com/elastic/elasticsearch/pull/125404) -* ES|QL `change_point` processing command [#120998](https://github.com/elastic/elasticsearch/pull/120998) -* Expose `input_type` option at root level for `text_embedding` task type in Perform Inference API [#122638](https://github.com/elastic/elasticsearch/pull/122638) (issue: [#122638](https://github.com/elastic/elasticsearch/pull/122638)) -* Integrate with `DeepSeek` API [#122218](https://github.com/elastic/elasticsearch/pull/122218) -* Limit the number of chunks for semantic text to prevent high memory usage [#123150](https://github.com/elastic/elasticsearch/pull/123150) -* Upgrade AWS v2 SDK to 2.30.38 [#124738](https://github.com/elastic/elasticsearch/pull/124738) -* [Inference API] Propagate product use case http header to EIS [#124025](https://github.com/elastic/elasticsearch/pull/124025) - -Mapping: -* Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. [#122615](https://github.com/elastic/elasticsearch/pull/122615) (issue: [#122615](https://github.com/elastic/elasticsearch/pull/122615)) -* Enable the use of nested field type with index.mode=time_series [#122224](https://github.com/elastic/elasticsearch/pull/122224) (issue: [#122224](https://github.com/elastic/elasticsearch/pull/122224)) -* Improved error message when index field type is invalid [#122860](https://github.com/elastic/elasticsearch/pull/122860) -* Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields [#119546](https://github.com/elastic/elasticsearch/pull/119546) -* Release semantic_text as a GA feature [#124669](https://github.com/elastic/elasticsearch/pull/124669) -* Store arrays offsets for boolean fields natively with synthetic source [#125529](https://github.com/elastic/elasticsearch/pull/125529) -* Store arrays offsets for ip fields natively with synthetic source [#122999](https://github.com/elastic/elasticsearch/pull/122999) -* Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. [#113757](https://github.com/elastic/elasticsearch/pull/113757) -* Store arrays offsets for numeric fields natively with synthetic source [#124594](https://github.com/elastic/elasticsearch/pull/124594) -* Store arrays offsets for unsigned long fields natively with synthetic source [#125709](https://github.com/elastic/elasticsearch/pull/125709) -* Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` [#124927](https://github.com/elastic/elasticsearch/pull/124927) -* Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields [#122637](https://github.com/elastic/elasticsearch/pull/122637) -* Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields [#124050](https://github.com/elastic/elasticsearch/pull/124050) -* Use `FallbackSyntheticSourceBlockLoader` for number fields [#122280](https://github.com/elastic/elasticsearch/pull/122280) - -Packaging: -* Update bundled JDK to Java 24 [#125159](https://github.com/elastic/elasticsearch/pull/125159) - -Ranking: -* Leverage scorer supplier in `QueryFeatureExtractor` [#125259](https://github.com/elastic/elasticsearch/pull/125259) - -Relevance: -* Skip semantic_text embedding generation when no content is provided. [#123763](https://github.com/elastic/elasticsearch/pull/123763) - -Search: -* Account for the `SearchHit` source in circuit breaker [#121920](https://github.com/elastic/elasticsearch/pull/121920) (issue: [#121920](https://github.com/elastic/elasticsearch/pull/121920)) -* Added optional parameters to QSTR ES|QL function [#121787](https://github.com/elastic/elasticsearch/pull/121787) (issue: [#121787](https://github.com/elastic/elasticsearch/pull/121787)) -* Optimize memory usage in `ShardBulkInferenceActionFilter` [#124313](https://github.com/elastic/elasticsearch/pull/124313) -* Optionally allow text similarity reranking to fail [#121784](https://github.com/elastic/elasticsearch/pull/121784) - -Security: -* Bump nimbus-jose-jwt to 10.0.2 [#124544](https://github.com/elastic/elasticsearch/pull/124544) - -Snapshot/Restore: -* GCS blob store: add `OperationPurpose/Operation` stats counters [#122991](https://github.com/elastic/elasticsearch/pull/122991) -* Retry when the server can't be resolved (Google Cloud Storage) [#123852](https://github.com/elastic/elasticsearch/pull/123852) -* Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) -* Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0 [#124062](https://github.com/elastic/elasticsearch/pull/124062) - -Stats: -* Run XPack usage actions on local node [#122933](https://github.com/elastic/elasticsearch/pull/122933) - -Store: -* Abort pending deletion on `IndicesService` close [#123569](https://github.com/elastic/elasticsearch/pull/123569) - -Vector Search: -* Add bit vector support to semantic text [#123187](https://github.com/elastic/elasticsearch/pull/123187) -* Add panama implementations of byte-bit and float-bit script operations [#124722](https://github.com/elastic/elasticsearch/pull/124722) (issue: [#124722](https://github.com/elastic/elasticsearch/pull/124722)) -* Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields [#122381](https://github.com/elastic/elasticsearch/pull/122381) (issue: [#122381](https://github.com/elastic/elasticsearch/pull/122381)) -* Allow zero for `rescore_vector.oversample` to indicate by-passing oversample and rescoring [#125599](https://github.com/elastic/elasticsearch/pull/125599) -* New `vector_rescore` parameter as a quantized index type option [#124581](https://github.com/elastic/elasticsearch/pull/124581) - -Watcher: -* Run `TransportGetWatcherSettingsAction` on local node [#122857](https://github.com/elastic/elasticsearch/pull/122857) - -### Fixes [elasticsearch-910-fixes] - -Aggregations: -* Aggs: Let terms run in global ords mode no match [#124782](https://github.com/elastic/elasticsearch/pull/124782) - -Allocation: -* `DesiredBalanceReconciler` always returns `AllocationStats` [#122458](https://github.com/elastic/elasticsearch/pull/122458) - -Analysis: -* Adjust exception thrown when unable to load hunspell dict [#123743](https://github.com/elastic/elasticsearch/pull/123743) -* Non existing synonyms sets do not fail shard recovery for indices [#125659](https://github.com/elastic/elasticsearch/pull/125659) (issue: [#125659](https://github.com/elastic/elasticsearch/pull/125659)) - -Cluster Coordination: -* Disable logging in `ClusterFormationFailureHelper` on shutdown [#125244](https://github.com/elastic/elasticsearch/pull/125244) (issue: [#125244](https://github.com/elastic/elasticsearch/pull/125244)) - -Data streams: -* Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover [#122905](https://github.com/elastic/elasticsearch/pull/122905) - -Distributed: -* Pass `IndexReshardingMetadata` over the wire [#124841](https://github.com/elastic/elasticsearch/pull/124841) - -Downsampling: -* Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. [#124451](https://github.com/elastic/elasticsearch/pull/124451) - -EQL: -* Fix EQL double invoking listener [#124918](https://github.com/elastic/elasticsearch/pull/124918) - -ES|QL: -* Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: [#122886](https://github.com/elastic/elasticsearch/pull/122886)) -* Avoid over collecting in Limit or Lucene Operator [#123296](https://github.com/elastic/elasticsearch/pull/123296) -* Catch parsing exception [#124958](https://github.com/elastic/elasticsearch/pull/124958) (issue: [#124958](https://github.com/elastic/elasticsearch/pull/124958)) -* Change the order of the optimization rules [#124335](https://github.com/elastic/elasticsearch/pull/124335) -* ESQL - date nanos range bug? [#125345](https://github.com/elastic/elasticsearch/pull/125345) (issue: [#125345](https://github.com/elastic/elasticsearch/pull/125345)) -* ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate` [#124446](https://github.com/elastic/elasticsearch/pull/124446) (issue: [#124446](https://github.com/elastic/elasticsearch/pull/124446)) -* ESQL: Fix inconsistent results in using scaled_float field [#122586](https://github.com/elastic/elasticsearch/pull/122586) (issue: [#122586](https://github.com/elastic/elasticsearch/pull/122586)) -* ESQL: Remove estimated row size assertion [#122762](https://github.com/elastic/elasticsearch/pull/122762) (issue: [#122762](https://github.com/elastic/elasticsearch/pull/122762)) -* ES|QL: Fix scoring for full text functions [#124540](https://github.com/elastic/elasticsearch/pull/124540) -* Esql - Fix lucene push down behavior when a range contains nanos and millis [#125595](https://github.com/elastic/elasticsearch/pull/125595) -* Fix early termination in `LuceneSourceOperator` [#123197](https://github.com/elastic/elasticsearch/pull/123197) -* Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: [#123492](https://github.com/elastic/elasticsearch/pull/123492)) -* Fix functions emitting warnings with no source [#122821](https://github.com/elastic/elasticsearch/pull/122821) (issue: [#122821](https://github.com/elastic/elasticsearch/pull/122821)) -* Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: [#122601](https://github.com/elastic/elasticsearch/pull/122601)) -* Improve error message for ( and [ [#124177](https://github.com/elastic/elasticsearch/pull/124177) (issue: [#124177](https://github.com/elastic/elasticsearch/pull/124177)) -* Lazy collection copying during node transform [#124424](https://github.com/elastic/elasticsearch/pull/124424) -* Make `numberOfChannels` consistent with layout map by removing duplicated `ChannelSet` [#125636](https://github.com/elastic/elasticsearch/pull/125636) -* Reduce iteration complexity for plan traversal [#123427](https://github.com/elastic/elasticsearch/pull/123427) -* Remove duplicated nested commands [#123085](https://github.com/elastic/elasticsearch/pull/123085) -* Revive inlinestats [#122257](https://github.com/elastic/elasticsearch/pull/122257) -* Revive some more of inlinestats functionality [#123589](https://github.com/elastic/elasticsearch/pull/123589) -* TO_LOWER processes all values [#124676](https://github.com/elastic/elasticsearch/pull/124676) (issue: [#124676](https://github.com/elastic/elasticsearch/pull/124676)) -* Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: [#124001](https://github.com/elastic/elasticsearch/pull/124001)) - -Engine: -* Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: [#123010](https://github.com/elastic/elasticsearch/pull/123010)) - -Highlighting: -* Restore V8 REST compatibility around highlight `force_source` parameter [#124873](https://github.com/elastic/elasticsearch/pull/124873) - -Indices APIs: -* Avoid hoarding cluster state references during rollover [#124107](https://github.com/elastic/elasticsearch/pull/124107) (issue: [#124107](https://github.com/elastic/elasticsearch/pull/124107)) -* Fix NPE in rolling over unknown target and return 404 [#125352](https://github.com/elastic/elasticsearch/pull/125352) -* Updates the deprecation info API to not warn about system indices and data streams [#122951](https://github.com/elastic/elasticsearch/pull/122951) - -Infra/Core: -* Fix system data streams to be restorable from a snapshot [#124651](https://github.com/elastic/elasticsearch/pull/124651) (issue: [#124651](https://github.com/elastic/elasticsearch/pull/124651)) -* Have create index return a bad request on poor formatting [#123761](https://github.com/elastic/elasticsearch/pull/123761) -* Include data streams when converting an existing resource to a system resource [#121392](https://github.com/elastic/elasticsearch/pull/121392) -* Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. [#124732](https://github.com/elastic/elasticsearch/pull/124732) (issue: [#124732](https://github.com/elastic/elasticsearch/pull/124732)) -* Reduce Data Loss in System Indices Migration [#121327](https://github.com/elastic/elasticsearch/pull/121327) -* System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) - -Ingest Node: -* Fix geoip databases index access after system feature migration (again) [#122938](https://github.com/elastic/elasticsearch/pull/122938) -* Fix geoip databases index access after system feature migration (take 3) [#124604](https://github.com/elastic/elasticsearch/pull/124604) -* apm-data: Use representative count as event.success_count if available [#119995](https://github.com/elastic/elasticsearch/pull/119995) - -Machine Learning: -* Add `ElasticInferenceServiceCompletionServiceSettings` [#123155](https://github.com/elastic/elasticsearch/pull/123155) -* Add enterprise license check to inference action for semantic text fields [#122293](https://github.com/elastic/elasticsearch/pull/122293) -* Avoid potentially throwing calls to Task#getDescription in model download [#124527](https://github.com/elastic/elasticsearch/pull/124527) -* Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` [#125023](https://github.com/elastic/elasticsearch/pull/125023) -* Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) -* Migrate `model_version` to `model_id` when parsing persistent elser inference endpoints [#124769](https://github.com/elastic/elasticsearch/pull/124769) (issue: [#124769](https://github.com/elastic/elasticsearch/pull/124769)) -* Provide model size statistics as soon as an anomaly detection job is opened [#124638](https://github.com/elastic/elasticsearch/pull/124638) (issue: [#124638](https://github.com/elastic/elasticsearch/pull/124638)) -* Retry on streaming errors [#123076](https://github.com/elastic/elasticsearch/pull/123076) -* Return a Conflict status code if the model deployment is stopped by a user [#125204](https://github.com/elastic/elasticsearch/pull/125204) (issue: [#125204](https://github.com/elastic/elasticsearch/pull/125204)) -* Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) -* Set default similarity for Cohere model to cosine [#125370](https://github.com/elastic/elasticsearch/pull/125370) (issue: [#125370](https://github.com/elastic/elasticsearch/pull/125370)) -* Updates to allow using Cohere binary embedding response in semantic search queries [#121827](https://github.com/elastic/elasticsearch/pull/121827) -* [Inference API] Fix output stream ordering in `InferenceActionProxy` [#124225](https://github.com/elastic/elasticsearch/pull/124225) - -Mapping: -* Avoid serializing empty `_source` fields in mappings [#122606](https://github.com/elastic/elasticsearch/pull/122606) -* Fix Semantic Text 8.x Upgrade Bug [#125446](https://github.com/elastic/elasticsearch/pull/125446) -* Merge template mappings properly during validation [#124784](https://github.com/elastic/elasticsearch/pull/124784) (issue: [#124784](https://github.com/elastic/elasticsearch/pull/124784)) - -Ranking: -* Fix LTR query feature with phrases (and two-phase) queries [#125103](https://github.com/elastic/elasticsearch/pull/125103) -* Restore `TextSimilarityRankBuilder` XContent output [#124564](https://github.com/elastic/elasticsearch/pull/124564) - -Relevance: -* Prevent Query Rule Creation with Invalid Numeric Match Criteria [#122823](https://github.com/elastic/elasticsearch/pull/122823) - -Search: -* Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions [#123890](https://github.com/elastic/elasticsearch/pull/123890) -* Don't generate stacktrace in `TaskCancelledException` [#125002](https://github.com/elastic/elasticsearch/pull/125002) -* Fix concurrency issue in `ScriptSortBuilder` [#123757](https://github.com/elastic/elasticsearch/pull/123757) -* Fix handling of auto expand replicas for stateless indices [#122365](https://github.com/elastic/elasticsearch/pull/122365) -* Handle long overflow in dates [#124048](https://github.com/elastic/elasticsearch/pull/124048) (issue: [#124048](https://github.com/elastic/elasticsearch/pull/124048)) -* Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: [#122357](https://github.com/elastic/elasticsearch/pull/122357)) -* Let MLTQuery throw IAE when no analyzer is set [#124662](https://github.com/elastic/elasticsearch/pull/124662) (issue: [#124662](https://github.com/elastic/elasticsearch/pull/124662)) -* Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` [#125650](https://github.com/elastic/elasticsearch/pull/125650) (issue: [#125650](https://github.com/elastic/elasticsearch/pull/125650)) -* Minor-Fixes Support 7x segments as archive in 8x / 9x [#125666](https://github.com/elastic/elasticsearch/pull/125666) -* Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. [#119503](https://github.com/elastic/elasticsearch/pull/119503) (issue: [#119503](https://github.com/elastic/elasticsearch/pull/119503)) - -Snapshot/Restore: -* Fork post-snapshot-delete cleanup off master thread [#122731](https://github.com/elastic/elasticsearch/pull/122731) -* Limit number of suppressed S3 deletion errors [#123630](https://github.com/elastic/elasticsearch/pull/123630) (issue: [#123630](https://github.com/elastic/elasticsearch/pull/123630)) -* This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. [#124931](https://github.com/elastic/elasticsearch/pull/124931) - Suggesters: * Return an empty suggestion when suggest phase times out [#122575](https://github.com/elastic/elasticsearch/pull/122575) (issue: [#122575](https://github.com/elastic/elasticsearch/pull/122575)) * Support duplicate suggestions in completion field [#121324](https://github.com/elastic/elasticsearch/pull/121324) (issue: [#121324](https://github.com/elastic/elasticsearch/pull/121324)) @@ -781,6 +685,11 @@ Transform: * If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias [#122074](https://github.com/elastic/elasticsearch/pull/122074) (issue: [#122074](https://github.com/elastic/elasticsearch/pull/122074)) Vector Search: +* Apply default k for knn query eagerly [#118774](https://github.com/elastic/elasticsearch/pull/118774) +* Fix `bbq_hnsw` merge file cleanup on random IO exceptions [#119691](https://github.com/elastic/elasticsearch/pull/119691) (issue: [#119691](https://github.com/elastic/elasticsearch/pull/119691)) * Knn vector rescoring to sort score docs [#122653](https://github.com/elastic/elasticsearch/pull/122653) (issue: [#122653](https://github.com/elastic/elasticsearch/pull/122653)) * Return appropriate error on null dims update instead of npe [#125716](https://github.com/elastic/elasticsearch/pull/125716) +Watcher: +* Watcher history index has too many indexed fields - [#117701](https://github.com/elastic/elasticsearch/pull/117701) (issue: [#117701](https://github.com/elastic/elasticsearch/pull/117701)) + From 65413dd550a302ef6551bd6d7bfbe011b4ce6ecd Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 14 Apr 2025 14:21:23 -0400 Subject: [PATCH 16/34] Remove 9.1.0 release notes from 9.0.0 --- .../release/BundleChangelogsTask.java | 3 +- docs/release-notes/breaking-changes.md | 3 - .../release-notes/changelog-bundles/9.0.0.yml | 610 +-------- .../release-notes/changelog-bundles/9.1.0.yml | 1097 ----------------- docs/release-notes/index.md | 125 -- 5 files changed, 3 insertions(+), 1835 deletions(-) delete mode 100644 docs/release-notes/changelog-bundles/9.1.0.yml diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index 14bd20a1e328b..4a7fa8d1f2ad3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -74,7 +74,7 @@ public BundleChangelogsTask(ObjectFactory objectFactory, ExecOperations execOper public void executeTask() throws IOException { final String upstreamRemote = gitWrapper.getUpstream(); - String ref = "43827b943938345d348826f5766dd8e986ec7b20"; + String ref = "9.0-fix-release-notes"; try { checkoutChangelogs(gitWrapper, upstreamRemote, ref); Properties props = new Properties(); @@ -101,6 +101,7 @@ public void executeTask() throws IOException { private static void checkoutChangelogs(GitWrapper gitWrapper, String upstream, String ref) { gitWrapper.updateRemote(upstream); // TODO check for changes first + gitWrapper.runCommand("rm", "-rf", "docs/changelog"); gitWrapper.runCommand("git", "checkout", ref, "--", "docs/changelog"); } diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index 9c6b3122957e3..a80e6fa606661 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -37,9 +37,6 @@ will prevent node from starting [#118366](https://github.com/elastic/elasticsear Cluster Coordination: * Remove unsupported legacy value for `discovery.type` [#112903](https://github.com/elastic/elasticsearch/pull/112903) -Discovery-Plugins: -* Upgrade `discovery-ec2` to AWS SDK v2 [#122062](https://github.com/elastic/elasticsearch/pull/122062) - EQL: * Set allow_partial_search_results=true by default [#120267](https://github.com/elastic/elasticsearch/pull/120267) diff --git a/docs/release-notes/changelog-bundles/9.0.0.yml b/docs/release-notes/changelog-bundles/9.0.0.yml index efbd24564c697..1f1d1aac2847c 100644 --- a/docs/release-notes/changelog-bundles/9.0.0.yml +++ b/docs/release-notes/changelog-bundles/9.0.0.yml @@ -1,5 +1,5 @@ version: 9.0.0 -generated: 2025-04-07T17:44:21.076873Z +generated: 2025-04-14T18:16:08.465456Z changelogs: - pr: 90529 summary: Output a consistent format when generating error json @@ -40,12 +40,6 @@ changelogs: impact: "To continue to use machine learning functionality on macOS please switch to an arm64 machine (Apple silicon). Alternatively, it will still be possible to run Elasticsearch with machine learning enabled in a Docker container on macOS x86_64." notable: false essSettingChange: false - - pr: 106953 - summary: Optimize usage calculation in ILM policies retrieval API - area: ILM+SLM - type: enhancement - issues: - - 105773 - pr: 111104 summary: "ESQL: Enable async get to support formatting" area: ES|QL @@ -175,11 +169,6 @@ changelogs: impact: Replace usages of "german2" with "german" in analysis configuration. Old indices that use the "german" stemmer should be reindexed if possible. notable: false essSettingChange: false - - pr: 113757 - summary: Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. - area: Mapping - type: enhancement - issues: [] - pr: 113827 summary: Add Optional Source Filtering to Source Loaders area: Mapping @@ -705,11 +694,6 @@ changelogs: issues: - 116617 - 116618 - - pr: 117642 - summary: Adding endpoint creation validation to `ElasticInferenceService` - area: Machine Learning - type: enhancement - issues: [] - pr: 117643 summary: Drop null columns in text formats area: ES|QL @@ -1305,11 +1289,6 @@ changelogs: type: bug issues: - 111312 - - pr: 119546 - summary: Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields - area: Mapping - type: enhancement - issues: [] - pr: 119564 summary: Http stream activity tracker and exceptions handling area: Network @@ -1460,11 +1439,6 @@ changelogs: area: Allocation type: enhancement issues: [] - - pr: 119995 - summary: "apm-data: Use representative count as event.success_count if available" - area: Ingest Node - type: bug - issues: [] - pr: 120020 summary: Resume Driver on cancelled or early finished area: ES|QL @@ -1788,11 +1762,6 @@ changelogs: impact: Users should stop setting this field (which did not have any effect anyway). notable: false essSettingChange: false - - pr: 120751 - summary: Adding support for binary embedding type to Cohere service embedding type - area: Machine Learning - type: enhancement - issues: [] - pr: 120752 summary: Refresh source index before reindexing data stream index area: Data streams @@ -1804,11 +1773,6 @@ changelogs: area: Ingest Node type: enhancement issues: [] - - pr: 120774 - summary: Retry ES|QL node requests on shard level failures - area: ES|QL - type: enhancement - issues: [] - pr: 120799 summary: Remove duplicate code in ESIntegTestCase area: Search @@ -1868,11 +1832,6 @@ changelogs: area: ES|QL type: bug issues: [] - - pr: 120869 - summary: Threadpool merge scheduler - area: Engine - type: feature - issues: [] - pr: 120883 summary: Add `?master_timeout` to `POST /_ilm/migrate_to_data_tiers` area: Indices APIs @@ -1889,11 +1848,6 @@ changelogs: type: bug issues: - 120518 - - pr: 120957 - summary: Introduce `AllocationBalancingRoundSummaryService` - area: Allocation - type: enhancement - issues: [] - pr: 120959 summary: Remove unnecessary entitlement area: Infra/Plugins @@ -1910,11 +1864,6 @@ changelogs: area: Authentication type: enhancement issues: [] - - pr: 120998 - summary: ES|QL `change_point` processing command - area: Machine Learning - type: feature - issues: [] - pr: 121048 summary: Updating Inference Update API documentation to have the correct PUT method area: Machine Learning @@ -1949,11 +1898,6 @@ changelogs: area: Vector Search type: feature issues: [] - - pr: 121106 - summary: Add `ModelRegistryMetadata` to Cluster State - area: Machine Learning - type: enhancement - issues: [] - pr: 121109 summary: Fix propagation of dynamic mapping parameter when applying `copy_to` area: Mapping @@ -1999,42 +1943,11 @@ changelogs: area: Inference type: bug issues: [] - - pr: 121240 - summary: Implement runtime skip_unavailable=true - area: ES|QL - type: enhancement - issues: [] - - pr: 121256 - summary: Run `TransportEnrichStatsAction` on local node - area: Ingest Node - type: enhancement - issues: [] - - pr: 121260 - summary: Introduce a pre-mapping logical plan processing step - area: ES|QL - type: enhancement - issues: [] - - pr: 121324 - summary: Support duplicate suggestions in completion field - area: Suggesters - type: bug - issues: - - 82432 - pr: 121325 summary: '`ReindexDataStreamIndex` bug in assertion caused by reference equality' area: Data streams type: bug issues: [] - - pr: 121327 - summary: Reduce Data Loss in System Indices Migration - area: Infra/Core - type: bug - issues: [] - - pr: 121370 - summary: Improve SLM Health Indicator to cover missing snapshot - area: ILM+SLM - type: enhancement - issues: [] - pr: 121392 summary: Include data streams when converting an existing resource to a system resource area: Infra/Core @@ -2045,11 +1958,6 @@ changelogs: area: Machine Learning type: bug issues: [] - - pr: 121548 - summary: Adding support for specifying embedding type to Jina AI service settings - area: Machine Learning - type: enhancement - issues: [] - pr: 121552 summary: Fix a bug in TOP area: ES|QL @@ -2103,11 +2011,6 @@ changelogs: impact: "Most users will not be impacted. If your Elastisearch 8 cluster was using a custom JDK and you relied on TLSv1.1, then you will need to explicitly enable TLSv1.1 within Elasticsearch (as well as enabling it within your JDK)" notable: false essSettingChange: false - - pr: 121784 - summary: Optionally allow text similarity reranking to fail - area: Search - type: enhancement - issues: [] - pr: 121787 summary: Added optional parameters to QSTR ES|QL function area: Search @@ -2119,21 +2022,11 @@ changelogs: area: ES|QL type: enhancement issues: [] - - pr: 121805 - summary: Support subset of metrics in aggregate metric double - area: ES|QL - type: enhancement - issues: [] - pr: 121821 summary: Fix get all inference endponts not returning multiple endpoints sharing model deployment area: Machine Learning type: bug issues: [] - - pr: 121827 - summary: Updates to allow using Cohere binary embedding response in semantic search queries - area: Machine Learning - type: bug - issues: [] - pr: 121843 summary: Fix async stop sometimes not properly collecting result area: ES|QL @@ -2145,100 +2038,17 @@ changelogs: area: ES|QL type: enhancement issues: [] - - pr: 121885 - summary: Introduce batched query execution and data-node side reduce - area: Search - type: enhancement - issues: [] - - pr: 121920 - summary: Account for the `SearchHit` source in circuit breaker - area: Search - type: enhancement - issues: - - 89656 - - pr: 121942 - summary: Allow partial results in ES|QL - area: ES|QL - type: enhancement - issues: [] - - pr: 121948 - summary: Add initial grammar and changes for FORK - area: ES|QL - type: feature - issues: [] - pr: 121971 summary: Do not fetch reserved roles from native store when Get Role API is called area: Authorization type: enhancement issues: [] - - pr: 122062 - summary: Upgrade `discovery-ec2` to AWS SDK v2 - area: Discovery-Plugins - type: breaking - issues: [] - breaking: - area: Cluster and node setting - title: Upgrade `discovery-ec2` to AWS SDK v2 - details: |2- - - In earlier versions of {es} the `discovery-ec2` plugin was based on the AWS SDK v1. AWS will withdraw support for this SDK before the end of the life of {es} {minor-version} so we must migrate to the newer AWS SDK v2. - Unfortunately there are several differences between the two AWS SDK versions which may require you to adjust your system configuration when upgrading to {es} {minor-version} or later. These differences include, but may not be limited to, the following items. - * AWS SDK v2 does not support the EC2 IMDSv1 protocol. - * AWS SDK v2 does not support the `aws.secretKey` or - `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties. - - * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so - the `discovery.ec2.protocol` setting is no longer effective. - - * AWS SDK v2 does not accept an access key without a secret key or vice - versa. - impact: |2- - - If you use the `discovery-ec2` plugin, test your upgrade thoroughly before upgrading any production workloads. - Adapt your configuration to the new SDK functionality. This includes, but may not be limited to, the following items. - * If you use IMDS to determine the availability zone of a node or to obtain - credentials for accessing the EC2 API, ensure that it supports the IMDSv2 - protocol. - - * If applicable, discontinue use of the `aws.secretKey` and - `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties. - - * If applicable, specify that you wish to use the insecure HTTP protocol to - access the EC2 API by setting `discovery.ec2.endpoint` to a URL which - starts with `http://`. - - * Either supply both an access key and a secret key using the keystore - settings `discovery.ec2.access_key` and `discovery.ec2.secret_key`, or - configure neither of these settings. - notable: true - essSettingChange: false - - pr: 122066 - summary: Adding elser default endpoint for EIS - area: Machine Learning - type: enhancement - issues: [] - pr: 122074 summary: "If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias" area: Transform type: bug issues: - 121913 - - pr: 122134 - summary: Adding integration for VoyageAI embeddings and rerank models - area: Machine Learning - type: enhancement - issues: [] - - pr: 122218 - summary: Integrate with `DeepSeek` API - area: Machine Learning - type: enhancement - issues: [] - - pr: 122224 - summary: Enable the use of nested field type with index.mode=time_series - area: Mapping - type: enhancement - issues: - - 120874 - pr: 122246 summary: Ensure removal of index blocks does not leave key with null value area: Data streams @@ -2255,21 +2065,11 @@ changelogs: area: ES|QL type: bug issues: [] - - pr: 122272 - summary: "[Inference API] Rename `model_id` prop to model in EIS sparse inference request body" - area: Inference - type: enhancement - issues: [] - pr: 122278 summary: Fix serialising the inference update request area: Machine Learning type: bug issues: [] - - pr: 122280 - summary: Use `FallbackSyntheticSourceBlockLoader` for number fields - area: Mapping - type: enhancement - issues: [] - pr: 122293 summary: Add enterprise license check to inference action for semantic text fields area: Machine Learning @@ -2286,22 +2086,6 @@ changelogs: type: bug issues: - 122186 - - pr: 122365 - summary: Fix handling of auto expand replicas for stateless indices - area: Search - type: bug - issues: [] - - pr: 122381 - summary: Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields - area: Vector Search - type: enhancement - issues: - - 117274 - - pr: 122390 - summary: Add health indicator impact to `HealthPeriodicLogger` - area: Health - type: enhancement - issues: [] - pr: 122409 summary: Allow setting the `type` in the reroute processor area: Ingest Node @@ -2318,33 +2102,12 @@ changelogs: area: Snapshot/Restore type: upgrade issues: [] - - pr: 122458 - summary: '`DesiredBalanceReconciler` always returns `AllocationStats`' - area: Allocation - type: bug - issues: [] - - pr: 122459 - summary: Double parameter markers for identifiers - area: ES|QL - type: enhancement - issues: [] - - pr: 122486 - summary: Add index mode to get data stream API - area: Data streams - type: enhancement - issues: [] - pr: 122575 summary: Return an empty suggestion when suggest phase times out area: Suggesters type: bug issues: - 122548 - - pr: 122586 - summary: "ESQL: Fix inconsistent results in using scaled_float field" - area: ES|QL - type: bug - issues: - - 122547 - pr: 122601 summary: Implicit numeric casting for CASE/GREATEST/LEAST area: ES|QL @@ -2356,39 +2119,12 @@ changelogs: area: Mapping type: bug issues: [] - - pr: 122615 - summary: Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. - area: Mapping - type: enhancement - issues: - - 116726 - - pr: 122637 - summary: Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields - area: Mapping - type: enhancement - issues: [] - - pr: 122638 - summary: Expose `input_type` option at root level for `text_embedding` task type in Perform Inference API - area: Machine Learning - type: enhancement - issues: - - 117856 - pr: 122653 summary: Knn vector rescoring to sort score docs area: Vector Search type: bug issues: - 119711 - - pr: 122660 - summary: Render `aggregate_metric_double` - area: ES|QL - type: enhancement - issues: [] - - pr: 122708 - summary: Support partial results in CCS in ES|QL - area: ES|QL - type: enhancement - issues: [] - pr: 122731 summary: Fork post-snapshot-delete cleanup off master thread area: Snapshot/Restore @@ -2400,68 +2136,17 @@ changelogs: type: bug issues: - 121535 - - pr: 122821 - summary: Fix functions emitting warnings with no source - area: ES|QL - type: bug - issues: - - 122588 - - pr: 122823 - summary: Prevent Query Rule Creation with Invalid Numeric Match Criteria - area: Relevance - type: bug - issues: [] - - pr: 122852 - summary: Run `TransportGetDataStreamsAction` on local node - area: Data streams - type: enhancement - issues: [] - - pr: 122857 - summary: Run `TransportGetWatcherSettingsAction` on local node - area: Watcher - type: enhancement - issues: [] - - pr: 122860 - summary: Improved error message when index field type is invalid - area: Mapping - type: enhancement - issues: [] - - pr: 122885 - summary: Run `TransportExplainLifecycleAction` on local node - area: ILM+SLM - type: enhancement - issues: [] - pr: 122886 summary: Add support to VALUES aggregation for spatial types area: ES|QL type: bug issues: - 122413 - - pr: 122890 - summary: Introduce `allow_partial_results` setting in ES|QL - area: ES|QL - type: enhancement - issues: [] - - pr: 122891 - summary: Pragma to load from stored fields - area: ES|QL - type: enhancement - issues: [] - pr: 122905 summary: Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover area: Data streams type: bug issues: [] - - pr: 122921 - summary: Run `TransportGetMappingsAction` on local node - area: Indices APIs - type: enhancement - issues: [] - - pr: 122933 - summary: Run XPack usage actions on local node - area: Stats - type: enhancement - issues: [] - pr: 122938 summary: Fix geoip databases index access after system feature migration (again) area: Ingest Node @@ -2484,16 +2169,6 @@ changelogs: impact: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. notable: false essSettingChange: false - - pr: 122991 - summary: "GCS blob store: add `OperationPurpose/Operation` stats counters" - area: Snapshot/Restore - type: enhancement - issues: [] - - pr: 122999 - summary: Store arrays offsets for ip fields natively with synthetic source - area: Mapping - type: enhancement - issues: [] - pr: 123010 summary: Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` area: Engine @@ -2505,26 +2180,11 @@ changelogs: area: Machine Learning type: bug issues: [] - - pr: 123085 - summary: Remove duplicated nested commands - area: ES|QL - type: bug - issues: [] - - pr: 123150 - summary: Limit the number of chunks for semantic text to prevent high memory usage - area: Machine Learning - type: feature - issues: [] - pr: 123155 summary: Add `ElasticInferenceServiceCompletionServiceSettings` area: Machine Learning type: bug issues: [] - - pr: 123187 - summary: Add bit vector support to semantic text - area: Vector Search - type: enhancement - issues: [] - pr: 123272 summary: Set Connect Timeout to 5s area: Machine Learning @@ -2547,43 +2207,17 @@ changelogs: type: enhancement issues: - 123067 - - pr: 123396 - summary: Add initial grammar and planning for RRF (snapshot) - area: ES|QL - type: feature - issues: [] - pr: 123427 summary: Reduce iteration complexity for plan traversal area: ES|QL type: bug issues: [] - - pr: 123460 - summary: "ES|QL: Support `::date` in inline cast" - area: ES|QL - type: enhancement - issues: - - 116746 - pr: 123492 summary: Fix function registry concurrency issues on constructor area: ES|QL type: bug issues: - 123430 - - pr: 123543 - summary: Enhance memory accounting for document expansion and introduce max document size limit - area: CRUD - type: enhancement - issues: [] - - pr: 123569 - summary: Abort pending deletion on `IndicesService` close - area: Store - type: enhancement - issues: [] - - pr: 123588 - summary: Give Kibana user 'all' permissions for .entity_analytics.* indices - area: Infra/Core - type: enhancement - issues: [] - pr: 123589 summary: Revive some more of inlinestats functionality area: ES|QL @@ -2601,17 +2235,6 @@ changelogs: impact: "The dropped ciphers are `TLS_RSA_WITH_AES_256_GCM_SHA384`, `TLS_RSA_WITH_AES_128_GCM_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA`, and `TLS_RSA_WITH_AES_128_CBC_SHA`. TLS connections to Elasticsearch using these ciphers will no longer work. Please configure your clients to use one of supported cipher suites." notable: false essSettingChange: false - - pr: 123630 - summary: Limit number of suppressed S3 deletion errors - area: Snapshot/Restore - type: bug - issues: - - 123354 - - pr: 123712 - summary: Process ILM cluster state updates on another thread - area: ILM+SLM - type: enhancement - issues: [] - pr: 123728 summary: Allow skip shards with `_tier` and `_index` in ES|QL area: ES|QL @@ -2627,64 +2250,28 @@ changelogs: area: Infra/Core type: bug issues: [] - - pr: 123763 - summary: Skip semantic_text embedding generation when no content is provided. - area: Relevance - type: enhancement - issues: [] - - pr: 123852 - summary: Retry when the server can't be resolved (Google Cloud Storage) - area: Snapshot/Restore - type: enhancement - issues: [] - pr: 124001 summary: Use a must boolean statement when pushing down to Lucene when scoring is also needed area: ES|QL type: bug issues: - 123967 - - pr: 124025 - summary: "[Inference API] Propagate product use case http header to EIS" - area: Machine Learning - type: enhancement - issues: [] - pr: 124048 summary: Handle long overflow in dates area: Search type: bug issues: - 112483 - - pr: 124050 - summary: Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields - area: Mapping - type: enhancement - issues: [] - - pr: 124094 - summary: ES|QL slow log - area: ES|QL - type: enhancement - issues: [] - pr: 124149 summary: Retry ILM async action after reindexing data stream area: Data streams type: enhancement issues: [] - - pr: 124177 - summary: "Improve error message for ( and [" - area: ES|QL - type: bug - issues: - - 124145 - pr: 124225 summary: "[Inference API] Fix output stream ordering in `InferenceActionProxy`" area: Machine Learning type: bug issues: [] - - pr: 124313 - summary: Optimize memory usage in `ShardBulkInferenceActionFilter` - area: Search - type: enhancement - issues: [] - pr: 124335 summary: Change the order of the optimization rules area: ES|QL @@ -2695,11 +2282,6 @@ changelogs: area: Data streams type: enhancement issues: [] - - pr: 124394 - summary: Avoid `NamedWritable` in block serialization - area: ES|QL - type: enhancement - issues: [] - pr: 124424 summary: Lazy collection copying during node transform area: ES|QL @@ -2731,52 +2313,16 @@ changelogs: area: ES|QL type: bug issues: [] - - pr: 124564 - summary: Restore `TextSimilarityRankBuilder` XContent output - area: Ranking - type: bug - issues: [] - - pr: 124574 - summary: Allow passing several reserved state chunks in single process call - area: Infra/Settings - type: enhancement - issues: [] - - pr: 124581 - summary: New `vector_rescore` parameter as a quantized index type option - area: Vector Search - type: enhancement - issues: [] - - pr: 124594 - summary: Store arrays offsets for numeric fields natively with synthetic source - area: Mapping - type: enhancement - issues: [] - - pr: 124595 - summary: '`ToAggregateMetricDouble` function' - area: ES|QL - type: enhancement - issues: [] - pr: 124604 summary: Fix geoip databases index access after system feature migration (take 3) area: Ingest Node type: bug issues: [] - - pr: 124610 - summary: Remove page alignment in exchange sink - area: ES|QL - type: enhancement - issues: [] - pr: 124611 summary: Reuse child `outputSet` inside the plan where possible area: ES|QL type: enhancement issues: [] - - pr: 124638 - summary: Provide model size statistics as soon as an anomaly detection job is opened - area: Machine Learning - type: bug - issues: - - 121168 - pr: 124651 summary: Fix system data streams to be restorable from a snapshot area: Infra/Core @@ -2799,28 +2345,6 @@ changelogs: title: Release semantic_text as a GA feature body: semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. pr: 124669 - - pr: 124676 - summary: TO_LOWER processes all values - area: ES|QL - type: bug - issues: - - 124002 - - pr: 124690 - summary: Avoid creating known_fields for every check in Alias - area: Indices APIs - type: enhancement - issues: [] - - pr: 124722 - summary: Add panama implementations of byte-bit and float-bit script operations - area: Vector Search - type: enhancement - issues: - - 117096 - - pr: 124738 - summary: Upgrade AWS v2 SDK to 2.30.38 - area: Machine Learning - type: upgrade - issues: [] - pr: 124739 summary: Improve rolling up metrics area: Downsampling @@ -2837,16 +2361,6 @@ changelogs: type: bug issues: - 123372 - - pr: 124823 - summary: Report failures on partial results - area: ES|QL - type: enhancement - issues: [] - - pr: 124841 - summary: Pass `IndexReshardingMetadata` over the wire - area: Distributed - type: bug - issues: [] - pr: 124843 summary: Ignore _JAVA_OPTIONS area: Infra/CLI @@ -2868,37 +2382,6 @@ changelogs: type: bug issues: - 122949 - - pr: 124898 - summary: Add cache support in `TransportGetAllocationStatsAction` - area: Allocation - type: enhancement - issues: - - 110716 - - pr: 124901 - summary: Calculate concurrent node limit - area: ES|QL - type: feature - issues: [] - - pr: 124913 - summary: Report `original_types` - area: ES|QL - type: enhancement - issues: [] - - pr: 124918 - summary: Fix EQL double invoking listener - area: EQL - type: bug - issues: [] - - pr: 124927 - summary: Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` - area: Mapping - type: enhancement - issues: [] - - pr: 124929 - summary: Include failures in partial response - area: ES|QL - type: enhancement - issues: [] - pr: 124931 summary: This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. area: Snapshot/Restore @@ -2914,22 +2397,11 @@ changelogs: area: Machine Learning type: bug issues: [] - - pr: 125054 - summary: Truncate `step_info` and error reason in ILM execution state and history - area: ILM+SLM - type: enhancement - issues: - - 124181 - pr: 125103 summary: Fix LTR query feature with phrases (and two-phase) queries area: Ranking type: bug issues: [] - - pr: 125117 - summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." - area: Infra/Core - type: upgrade - issues: [] - pr: 125159 summary: Update bundled JDK to Java 24 area: Packaging @@ -2940,44 +2412,6 @@ changelogs: area: Data streams type: enhancement issues: [] - - pr: 125191 - summary: Fix sorting when `aggregate_metric_double` present - area: ES|QL - type: enhancement - issues: [] - - pr: 125204 - summary: Return a Conflict status code if the model deployment is stopped by a user - area: Machine Learning - type: bug - issues: - - 123745 - - pr: 125213 - summary: Run `TransportGetDataStreamOptionsAction` on local node - area: Data streams - type: enhancement - issues: [] - - pr: 125214 - summary: Run `TransportGetDataStreamLifecycleAction` on local node - area: Data streams - type: enhancement - issues: [] - - pr: 125239 - summary: Adding common rerank options to Perform Inference API - area: Machine Learning - type: enhancement - issues: - - 111273 - - pr: 125244 - summary: Disable logging in `ClusterFormationFailureHelper` on shutdown - area: Cluster Coordination - type: bug - issues: - - 105559 - - pr: 125259 - summary: Leverage scorer supplier in `QueryFeatureExtractor` - area: Ranking - type: enhancement - issues: [] - pr: 125345 summary: ESQL - date nanos range bug? area: ES|QL @@ -2989,11 +2423,6 @@ changelogs: area: Indices APIs type: bug issues: [] - - pr: 125357 - summary: Keep ordinals in conversion functions - area: ES|QL - type: enhancement - issues: [] - pr: 125370 summary: Set default similarity for Cohere model to cosine area: Machine Learning @@ -3010,33 +2439,11 @@ changelogs: area: Mapping type: bug issues: [] - - pr: 125477 - summary: Prevent get datafeeds stats API returning an error when local tasks are slow to stop - area: Machine Learning - type: bug - issues: - - 104160 - - pr: 125520 - summary: Add `FailedShardEntry` info to shard-failed task source string - area: Allocation - type: enhancement - issues: - - 102606 - - pr: 125529 - summary: Store arrays offsets for boolean fields natively with synthetic source - area: Mapping - type: enhancement - issues: [] - pr: 125595 summary: Esql - Fix lucene push down behavior when a range contains nanos and millis area: ES|QL type: bug issues: [] - - pr: 125599 - summary: Allow zero for `rescore_vector.oversample` to indicate by-passing oversample and rescoring - area: Vector Search - type: enhancement - issues: [] - pr: 125606 summary: Rename deprecation index template area: Infra/Logging @@ -3072,16 +2479,6 @@ changelogs: area: Search type: bug issues: [] - - pr: 125690 - summary: Take double parameter markers for identifiers out of snapshot - area: ES|QL - type: enhancement - issues: [] - - pr: 125709 - summary: Store arrays offsets for unsigned long fields natively with synthetic source - area: Mapping - type: enhancement - issues: [] - pr: 125716 summary: Return appropriate error on null dims update instead of npe area: Vector Search @@ -3110,11 +2507,6 @@ changelogs: area: Search type: bug issues: [] - - pr: 125930 - summary: Infer the score mode to use from the Lucene collector - area: ES|QL - type: enhancement - issues: [] - pr: 126077 summary: Preventing `ConcurrentModificationException` when updating settings for more than one index area: Indices APIs diff --git a/docs/release-notes/changelog-bundles/9.1.0.yml b/docs/release-notes/changelog-bundles/9.1.0.yml deleted file mode 100644 index 1a6e3e52abdf3..0000000000000 --- a/docs/release-notes/changelog-bundles/9.1.0.yml +++ /dev/null @@ -1,1097 +0,0 @@ -version: 9.1.0 -generated: 2025-03-31T18:44:39.319035Z -changelogs: - - pr: 113757 - summary: Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. - area: Mapping - type: enhancement - issues: [] - - pr: 117642 - summary: Adding endpoint creation validation to `ElasticInferenceService` - area: Machine Learning - type: enhancement - issues: [] - - pr: 119503 - summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. - area: Search - type: bug - issues: - - 117042 - - pr: 119546 - summary: Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields - area: Mapping - type: enhancement - issues: [] - - pr: 119886 - summary: Initial support for unmapped fields - area: ES|QL - type: feature - issues: [] - - pr: 119995 - summary: "apm-data: Use representative count as event.success_count if available" - area: Ingest Node - type: bug - issues: [] - - pr: 120751 - summary: Adding support for binary embedding type to Cohere service embedding type - area: Machine Learning - type: enhancement - issues: [] - - pr: 120774 - summary: Retry ES|QL node requests on shard level failures - area: ES|QL - type: enhancement - issues: [] - - pr: 120869 - summary: Threadpool merge scheduler - area: Engine - type: feature - issues: [] - - pr: 120957 - summary: Introduce `AllocationBalancingRoundSummaryService` - area: Allocation - type: enhancement - issues: [] - - pr: 120998 - summary: ES|QL `change_point` processing command - area: Machine Learning - type: feature - issues: [] - - pr: 121106 - summary: Add `ModelRegistryMetadata` to Cluster State - area: Machine Learning - type: enhancement - issues: [] - - pr: 121240 - summary: Implement runtime skip_unavailable=true - area: ES|QL - type: enhancement - issues: [] - - pr: 121256 - summary: Run `TransportEnrichStatsAction` on local node - area: Ingest Node - type: enhancement - issues: [] - - pr: 121260 - summary: Introduce a pre-mapping logical plan processing step - area: ES|QL - type: enhancement - issues: [] - - pr: 121324 - summary: Support duplicate suggestions in completion field - area: Suggesters - type: bug - issues: - - 82432 - - pr: 121327 - summary: Reduce Data Loss in System Indices Migration - area: Infra/Core - type: bug - issues: [] - - pr: 121370 - summary: Improve SLM Health Indicator to cover missing snapshot - area: ILM+SLM - type: enhancement - issues: [] - - pr: 121392 - summary: Include data streams when converting an existing resource to a system resource - area: Infra/Core - type: bug - issues: [] - - pr: 121548 - summary: Adding support for specifying embedding type to Jina AI service settings - area: Machine Learning - type: enhancement - issues: [] - - pr: 121667 - summary: Add deprecation warning for flush API - area: Machine Learning - type: deprecation - issues: - - 121506 - deprecation: - area: REST API - title: Add deprecation warning for flush API - details: "The anomaly detection job flush API is deprecated since it is only required for the post data API, which was deprecated since 7.11.0." - impact: "This should have a minimal impact on users as the flush API is only required for the post data API, which was deprecated since 7.11.0." - notable: false - essSettingChange: false - - pr: 121784 - summary: Optionally allow text similarity reranking to fail - area: Search - type: enhancement - issues: [] - - pr: 121787 - summary: Added optional parameters to QSTR ES|QL function - area: Search - type: feature - issues: - - 120933 - - pr: 121793 - summary: ES|QL - Add scoring for full text functions disjunctions - area: ES|QL - type: enhancement - issues: [] - - pr: 121805 - summary: Support subset of metrics in aggregate metric double - area: ES|QL - type: enhancement - issues: [] - - pr: 121827 - summary: Updates to allow using Cohere binary embedding response in semantic search queries - area: Machine Learning - type: bug - issues: [] - - pr: 121920 - summary: Account for the `SearchHit` source in circuit breaker - area: Search - type: enhancement - issues: - - 89656 - - pr: 121942 - summary: Allow partial results in ES|QL - area: ES|QL - type: enhancement - issues: [] - - pr: 121948 - summary: Add initial grammar and changes for FORK - area: ES|QL - type: feature - issues: [] - - pr: 121971 - summary: Do not fetch reserved roles from native store when Get Role API is called - area: Authorization - type: enhancement - issues: [] - - pr: 122062 - summary: Upgrade `discovery-ec2` to AWS SDK v2 - area: Discovery-Plugins - type: breaking - issues: [] - breaking: - area: Cluster and node setting - title: Upgrade `discovery-ec2` to AWS SDK v2 - details: |2- - - In earlier versions of {es} the `discovery-ec2` plugin was based on the AWS SDK v1. AWS will withdraw support for this SDK before the end of the life of {es} {minor-version} so we must migrate to the newer AWS SDK v2. - Unfortunately there are several differences between the two AWS SDK versions which may require you to adjust your system configuration when upgrading to {es} {minor-version} or later. These differences include, but may not be limited to, the following items. - * AWS SDK v2 does not support the EC2 IMDSv1 protocol. - * AWS SDK v2 does not support the `aws.secretKey` or - `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties. - - * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so - the `discovery.ec2.protocol` setting is no longer effective. - - * AWS SDK v2 does not accept an access key without a secret key or vice - versa. - impact: |2- - - If you use the `discovery-ec2` plugin, test your upgrade thoroughly before upgrading any production workloads. - Adapt your configuration to the new SDK functionality. This includes, but may not be limited to, the following items. - * If you use IMDS to determine the availability zone of a node or to obtain - credentials for accessing the EC2 API, ensure that it supports the IMDSv2 - protocol. - - * If applicable, discontinue use of the `aws.secretKey` and - `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties. - - * If applicable, specify that you wish to use the insecure HTTP protocol to - access the EC2 API by setting `discovery.ec2.endpoint` to a URL which - starts with `http://`. - - * Either supply both an access key and a secret key using the keystore - settings `discovery.ec2.access_key` and `discovery.ec2.secret_key`, or - configure neither of these settings. - notable: true - essSettingChange: false - - pr: 122066 - summary: Adding elser default endpoint for EIS - area: Machine Learning - type: enhancement - issues: [] - - pr: 122074 - summary: "If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias" - area: Transform - type: bug - issues: - - 121913 - - pr: 122134 - summary: Adding integration for VoyageAI embeddings and rerank models - area: Machine Learning - type: enhancement - issues: [] - - pr: 122218 - summary: Integrate with `DeepSeek` API - area: Machine Learning - type: enhancement - issues: [] - - pr: 122224 - summary: Enable the use of nested field type with index.mode=time_series - area: Mapping - type: enhancement - issues: - - 120874 - - pr: 122250 - summary: "ESQL: Align `RENAME` behavior with `EVAL` for sequential processing" - area: ES|QL - type: enhancement - issues: - - 121739 - - pr: 122257 - summary: Revive inlinestats - area: ES|QL - type: bug - issues: [] - - pr: 122272 - summary: "[Inference API] Rename `model_id` prop to model in EIS sparse inference request body" - area: Inference - type: enhancement - issues: [] - - pr: 122278 - summary: Fix serialising the inference update request - area: Machine Learning - type: bug - issues: [] - - pr: 122280 - summary: Use `FallbackSyntheticSourceBlockLoader` for number fields - area: Mapping - type: enhancement - issues: [] - - pr: 122293 - summary: Add enterprise license check to inference action for semantic text fields - area: Machine Learning - type: bug - issues: [] - - pr: 122326 - summary: System Index Migration Failure Results in a Non-Recoverable State - area: Infra/Core - type: bug - issues: [] - - pr: 122357 - summary: Handle search timeout in `SuggestPhase` - area: Search - type: bug - issues: - - 122186 - - pr: 122365 - summary: Fix handling of auto expand replicas for stateless indices - area: Search - type: bug - issues: [] - - pr: 122381 - summary: Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields - area: Vector Search - type: enhancement - issues: - - 117274 - - pr: 122390 - summary: Add health indicator impact to `HealthPeriodicLogger` - area: Health - type: enhancement - issues: [] - - pr: 122409 - summary: Allow setting the `type` in the reroute processor - area: Ingest Node - type: enhancement - issues: - - 121553 - - pr: 122427 - summary: Improve size limiting string message - area: Infra/Core - type: enhancement - issues: [] - - pr: 122431 - summary: Upgrade AWS SDK to v1.12.746 - area: Snapshot/Restore - type: upgrade - issues: [] - - pr: 122458 - summary: '`DesiredBalanceReconciler` always returns `AllocationStats`' - area: Allocation - type: bug - issues: [] - - pr: 122459 - summary: Double parameter markers for identifiers - area: ES|QL - type: enhancement - issues: [] - - pr: 122486 - summary: Add index mode to get data stream API - area: Data streams - type: enhancement - issues: [] - - pr: 122575 - summary: Return an empty suggestion when suggest phase times out - area: Suggesters - type: bug - issues: - - 122548 - - pr: 122586 - summary: "ESQL: Fix inconsistent results in using scaled_float field" - area: ES|QL - type: bug - issues: - - 122547 - - pr: 122601 - summary: Implicit numeric casting for CASE/GREATEST/LEAST - area: ES|QL - type: bug - issues: - - 121890 - - pr: 122606 - summary: Avoid serializing empty `_source` fields in mappings - area: Mapping - type: bug - issues: [] - - pr: 122615 - summary: Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. - area: Mapping - type: enhancement - issues: - - 116726 - - pr: 122637 - summary: Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields - area: Mapping - type: enhancement - issues: [] - - pr: 122638 - summary: Expose `input_type` option at root level for `text_embedding` task type in Perform Inference API - area: Machine Learning - type: enhancement - issues: - - 117856 - - pr: 122653 - summary: Knn vector rescoring to sort score docs - area: Vector Search - type: bug - issues: - - 119711 - - pr: 122660 - summary: Render `aggregate_metric_double` - area: ES|QL - type: enhancement - issues: [] - - pr: 122708 - summary: Support partial results in CCS in ES|QL - area: ES|QL - type: enhancement - issues: [] - - pr: 122731 - summary: Fork post-snapshot-delete cleanup off master thread - area: Snapshot/Restore - type: bug - issues: [] - - pr: 122762 - summary: "ESQL: Remove estimated row size assertion" - area: ES|QL - type: bug - issues: - - 121535 - - pr: 122821 - summary: Fix functions emitting warnings with no source - area: ES|QL - type: bug - issues: - - 122588 - - pr: 122823 - summary: Prevent Query Rule Creation with Invalid Numeric Match Criteria - area: Relevance - type: bug - issues: [] - - pr: 122852 - summary: Run `TransportGetDataStreamsAction` on local node - area: Data streams - type: enhancement - issues: [] - - pr: 122857 - summary: Run `TransportGetWatcherSettingsAction` on local node - area: Watcher - type: enhancement - issues: [] - - pr: 122860 - summary: Improved error message when index field type is invalid - area: Mapping - type: enhancement - issues: [] - - pr: 122885 - summary: Run `TransportExplainLifecycleAction` on local node - area: ILM+SLM - type: enhancement - issues: [] - - pr: 122886 - summary: Add support to VALUES aggregation for spatial types - area: ES|QL - type: bug - issues: - - 122413 - - pr: 122890 - summary: Introduce `allow_partial_results` setting in ES|QL - area: ES|QL - type: enhancement - issues: [] - - pr: 122891 - summary: Pragma to load from stored fields - area: ES|QL - type: enhancement - issues: [] - - pr: 122905 - summary: Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover - area: Data streams - type: bug - issues: [] - - pr: 122921 - summary: Run `TransportGetMappingsAction` on local node - area: Indices APIs - type: enhancement - issues: [] - - pr: 122933 - summary: Run XPack usage actions on local node - area: Stats - type: enhancement - issues: [] - - pr: 122938 - summary: Fix geoip databases index access after system feature migration (again) - area: Ingest Node - type: bug - issues: [] - - pr: 122951 - summary: Updates the deprecation info API to not warn about system indices and data streams - area: Indices APIs - type: bug - issues: [] - - pr: 122960 - summary: Deprecate Behavioral Analytics CRUD apis - area: Search - type: deprecation - issues: [] - deprecation: - area: Search - title: Deprecate Behavioral Analytics CRUD apis - details: "Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. The APIs will still work for now, but will emit warning headers that the API has been deprecated." - impact: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. - notable: false - essSettingChange: false - - pr: 122991 - summary: "GCS blob store: add `OperationPurpose/Operation` stats counters" - area: Snapshot/Restore - type: enhancement - issues: [] - - pr: 122999 - summary: Store arrays offsets for ip fields natively with synthetic source - area: Mapping - type: enhancement - issues: [] - - pr: 123010 - summary: Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` - area: Engine - type: bug - issues: - - 122974 - - pr: 123076 - summary: Retry on streaming errors - area: Machine Learning - type: bug - issues: [] - - pr: 123085 - summary: Remove duplicated nested commands - area: ES|QL - type: bug - issues: [] - - pr: 123150 - summary: Limit the number of chunks for semantic text to prevent high memory usage - area: Machine Learning - type: feature - issues: [] - - pr: 123155 - summary: Add `ElasticInferenceServiceCompletionServiceSettings` - area: Machine Learning - type: bug - issues: [] - - pr: 123187 - summary: Add bit vector support to semantic text - area: Vector Search - type: enhancement - issues: [] - - pr: 123197 - summary: Fix early termination in `LuceneSourceOperator` - area: ES|QL - type: bug - issues: [] - - pr: 123272 - summary: Set Connect Timeout to 5s - area: Machine Learning - type: bug - issues: [] - - pr: 123290 - summary: Fix Driver status iterations and `cpuTime` - area: ES|QL - type: enhancement - issues: - - 122967 - - pr: 123296 - summary: Avoid over collecting in Limit or Lucene Operator - area: ES|QL - type: bug - issues: [] - - pr: 123381 - summary: Push down `StartsWith` and `EndsWith` functions to Lucene - area: ES|QL - type: enhancement - issues: - - 123067 - - pr: 123396 - summary: Add initial grammar and planning for RRF (snapshot) - area: ES|QL - type: feature - issues: [] - - pr: 123427 - summary: Reduce iteration complexity for plan traversal - area: ES|QL - type: bug - issues: [] - - pr: 123460 - summary: "ES|QL: Support `::date` in inline cast" - area: ES|QL - type: enhancement - issues: - - 116746 - - pr: 123492 - summary: Fix function registry concurrency issues on constructor - area: ES|QL - type: bug - issues: - - 123430 - - pr: 123543 - summary: Enhance memory accounting for document expansion and introduce max document size limit - area: CRUD - type: enhancement - issues: [] - - pr: 123569 - summary: Abort pending deletion on `IndicesService` close - area: Store - type: enhancement - issues: [] - - pr: 123588 - summary: Give Kibana user 'all' permissions for .entity_analytics.* indices - area: Infra/Core - type: enhancement - issues: [] - - pr: 123589 - summary: Revive some more of inlinestats functionality - area: ES|QL - type: bug - issues: [] - - pr: 123600 - summary: Drop `TLS_RSA` cipher support for JDK 24 - area: TLS - type: breaking - issues: [] - breaking: - area: Cluster and node setting - title: Drop `TLS_RSA` cipher support for JDK 24 - details: "This change removes `TLS_RSA` ciphers from the list of default supported ciphers, for Elasticsearch deployments running on JDK 24." - impact: "The dropped ciphers are `TLS_RSA_WITH_AES_256_GCM_SHA384`, `TLS_RSA_WITH_AES_128_GCM_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA`, and `TLS_RSA_WITH_AES_128_CBC_SHA`. TLS connections to Elasticsearch using these ciphers will no longer work. Please configure your clients to use one of supported cipher suites." - notable: false - essSettingChange: false - - pr: 123630 - summary: Limit number of suppressed S3 deletion errors - area: Snapshot/Restore - type: bug - issues: - - 123354 - - pr: 123712 - summary: Process ILM cluster state updates on another thread - area: ILM+SLM - type: enhancement - issues: [] - - pr: 123728 - summary: Allow skip shards with `_tier` and `_index` in ES|QL - area: ES|QL - type: enhancement - issues: [] - - pr: 123743 - summary: Adjust exception thrown when unable to load hunspell dict - area: Analysis - type: bug - issues: [] - - pr: 123757 - summary: Fix concurrency issue in `ScriptSortBuilder` - area: Search - type: bug - issues: [] - - pr: 123761 - summary: Have create index return a bad request on poor formatting - area: Infra/Core - type: bug - issues: [] - - pr: 123763 - summary: Skip semantic_text embedding generation when no content is provided. - area: Relevance - type: enhancement - issues: [] - - pr: 123852 - summary: Retry when the server can't be resolved (Google Cloud Storage) - area: Snapshot/Restore - type: enhancement - issues: [] - - pr: 123890 - summary: Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions - area: Search - type: bug - issues: [] - - pr: 124001 - summary: Use a must boolean statement when pushing down to Lucene when scoring is also needed - area: ES|QL - type: bug - issues: - - 123967 - - pr: 124025 - summary: "[Inference API] Propagate product use case http header to EIS" - area: Machine Learning - type: enhancement - issues: [] - - pr: 124048 - summary: Handle long overflow in dates - area: Search - type: bug - issues: - - 112483 - - pr: 124050 - summary: Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields - area: Mapping - type: enhancement - issues: [] - - pr: 124062 - summary: Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0 - area: Snapshot/Restore - type: upgrade - issues: [] - - pr: 124094 - summary: ES|QL slow log - area: ES|QL - type: enhancement - issues: [] - - pr: 124107 - summary: Avoid hoarding cluster state references during rollover - area: Indices APIs - type: bug - issues: - - 123893 - - pr: 124149 - summary: Retry ILM async action after reindexing data stream - area: Data streams - type: enhancement - issues: [] - - pr: 124177 - summary: "Improve error message for ( and [" - area: ES|QL - type: bug - issues: - - 124145 - - pr: 124225 - summary: "[Inference API] Fix output stream ordering in `InferenceActionProxy`" - area: Machine Learning - type: bug - issues: [] - - pr: 124313 - summary: Optimize memory usage in `ShardBulkInferenceActionFilter` - area: Search - type: enhancement - issues: [] - - pr: 124335 - summary: Change the order of the optimization rules - area: ES|QL - type: bug - issues: [] - - pr: 124363 - summary: Set cause on create index request in create from action - area: Data streams - type: enhancement - issues: [] - - pr: 124394 - summary: Avoid `NamedWritable` in block serialization - area: ES|QL - type: enhancement - issues: [] - - pr: 124424 - summary: Lazy collection copying during node transform - area: ES|QL - type: bug - issues: [] - - pr: 124446 - summary: "ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate`" - area: ES|QL - type: bug - issues: - - 124311 - - pr: 124451 - summary: Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. - area: Downsampling - type: bug - issues: [] - - pr: 124477 - summary: Improve downsample performance by buffering docids and do bulk processing - area: Downsampling - type: enhancement - issues: [] - - pr: 124527 - summary: Avoid potentially throwing calls to Task#getDescription in model download - area: Machine Learning - type: bug - issues: [] - - pr: 124540 - summary: "ES|QL: Fix scoring for full text functions" - area: ES|QL - type: bug - issues: [] - - pr: 124544 - summary: Bump nimbus-jose-jwt to 10.0.2 - area: Security - type: upgrade - issues: [] - - pr: 124564 - summary: Restore `TextSimilarityRankBuilder` XContent output - area: Ranking - type: bug - issues: [] - - pr: 124574 - summary: Allow passing several reserved state chunks in single process call - area: Infra/Settings - type: enhancement - issues: [] - - pr: 124581 - summary: New `vector_rescore` parameter as a quantized index type option - area: Vector Search - type: enhancement - issues: [] - - pr: 124594 - summary: Store arrays offsets for numeric fields natively with synthetic source - area: Mapping - type: enhancement - issues: [] - - pr: 124595 - summary: '`ToAggregateMetricDouble` function' - area: ES|QL - type: enhancement - issues: [] - - pr: 124604 - summary: Fix geoip databases index access after system feature migration (take 3) - area: Ingest Node - type: bug - issues: [] - - pr: 124610 - summary: Remove page alignment in exchange sink - area: ES|QL - type: enhancement - issues: [] - - pr: 124611 - summary: Reuse child `outputSet` inside the plan where possible - area: ES|QL - type: enhancement - issues: [] - - pr: 124638 - summary: Provide model size statistics as soon as an anomaly detection job is opened - area: Machine Learning - type: bug - issues: - - 121168 - - pr: 124651 - summary: Fix system data streams to be restorable from a snapshot - area: Infra/Core - type: bug - issues: - - 89261 - - pr: 124662 - summary: Let MLTQuery throw IAE when no analyzer is set - area: Search - type: bug - issues: - - 124562 - - pr: 124669 - summary: Release semantic_text as a GA feature - area: Mapping - type: feature - issues: [] - highlight: - notable: true - title: Release semantic_text as a GA feature - body: semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. - pr: 124669 - - pr: 124676 - summary: TO_LOWER processes all values - area: ES|QL - type: bug - issues: - - 124002 - - pr: 124690 - summary: Avoid creating known_fields for every check in Alias - area: Indices APIs - type: enhancement - issues: [] - - pr: 124722 - summary: Add panama implementations of byte-bit and float-bit script operations - area: Vector Search - type: enhancement - issues: - - 117096 - - pr: 124732 - summary: Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. - area: Infra/Core - type: bug - issues: - - 124667 - - pr: 124738 - summary: Upgrade AWS v2 SDK to 2.30.38 - area: Machine Learning - type: upgrade - issues: [] - - pr: 124739 - summary: Improve rolling up metrics - area: Downsampling - type: enhancement - issues: [] - - pr: 124769 - summary: Migrate `model_version` to `model_id` when parsing persistent elser inference endpoints - area: Machine Learning - type: bug - issues: - - 124675 - - pr: 124782 - summary: "Aggs: Let terms run in global ords mode no match" - area: Aggregations - type: bug - issues: [] - - pr: 124784 - summary: Merge template mappings properly during validation - area: Mapping - type: bug - issues: - - 123372 - - pr: 124823 - summary: Report failures on partial results - area: ES|QL - type: enhancement - issues: [] - - pr: 124841 - summary: Pass `IndexReshardingMetadata` over the wire - area: Distributed - type: bug - issues: [] - - pr: 124843 - summary: Ignore _JAVA_OPTIONS - area: Infra/CLI - type: enhancement - issues: [] - - pr: 124873 - summary: Restore V8 REST compatibility around highlight `force_source` parameter - area: Highlighting - type: bug - issues: [] - - pr: 124898 - summary: Add cache support in `TransportGetAllocationStatsAction` - area: Allocation - type: enhancement - issues: - - 110716 - - pr: 124901 - summary: Calculate concurrent node limit - area: ES|QL - type: feature - issues: [] - - pr: 124913 - summary: Report `original_types` - area: ES|QL - type: enhancement - issues: [] - - pr: 124918 - summary: Fix EQL double invoking listener - area: EQL - type: bug - issues: [] - - pr: 124927 - summary: Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` - area: Mapping - type: enhancement - issues: [] - - pr: 124929 - summary: Include failures in partial response - area: ES|QL - type: enhancement - issues: [] - - pr: 124931 - summary: This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. - area: Snapshot/Restore - type: bug - issues: [] - - pr: 124936 - summary: Indicate when errors represent timeouts - area: Infra/REST API - type: enhancement - issues: [] - - pr: 124958 - summary: Catch parsing exception - area: ES|QL - type: bug - issues: - - 119025 - - pr: 125002 - summary: Don't generate stacktrace in `TaskCancelledException` - area: Search - type: bug - issues: [] - - pr: 125023 - summary: Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` - area: Machine Learning - type: bug - issues: [] - - pr: 125054 - summary: Truncate `step_info` and error reason in ILM execution state and history - area: ILM+SLM - type: enhancement - issues: - - 124181 - - pr: 125103 - summary: Fix LTR query feature with phrases (and two-phase) queries - area: Ranking - type: bug - issues: [] - - pr: 125117 - summary: "Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager." - area: Infra/Core - type: upgrade - issues: [] - - pr: 125159 - summary: Update bundled JDK to Java 24 - area: Packaging - type: upgrade - issues: [] - - pr: 125171 - summary: Reindex data stream indices on different nodes - area: Data streams - type: enhancement - issues: [] - - pr: 125204 - summary: Return a Conflict status code if the model deployment is stopped by a user - area: Machine Learning - type: bug - issues: - - 123745 - - pr: 125213 - summary: Run `TransportGetDataStreamOptionsAction` on local node - area: Data streams - type: enhancement - issues: [] - - pr: 125214 - summary: Run `TransportGetDataStreamLifecycleAction` on local node - area: Data streams - type: enhancement - issues: [] - - pr: 125239 - summary: Adding common rerank options to Perform Inference API - area: Machine Learning - type: enhancement - issues: - - 111273 - - pr: 125244 - summary: Disable logging in `ClusterFormationFailureHelper` on shutdown - area: Cluster Coordination - type: bug - issues: - - 105559 - - pr: 125259 - summary: Leverage scorer supplier in `QueryFeatureExtractor` - area: Ranking - type: enhancement - issues: [] - - pr: 125345 - summary: ESQL - date nanos range bug? - area: ES|QL - type: bug - issues: - - 125439 - - pr: 125352 - summary: Fix NPE in rolling over unknown target and return 404 - area: Indices APIs - type: bug - issues: [] - - pr: 125357 - summary: Keep ordinals in conversion functions - area: ES|QL - type: enhancement - issues: [] - - pr: 125370 - summary: Set default similarity for Cohere model to cosine - area: Machine Learning - type: bug - issues: - - 122878 - - pr: 125404 - summary: Check if the anomaly results index has been rolled over - area: Machine Learning - type: upgrade - issues: [] - - pr: 125446 - summary: Fix Semantic Text 8.x Upgrade Bug - area: Mapping - type: bug - issues: [] - - pr: 125529 - summary: Store arrays offsets for boolean fields natively with synthetic source - area: Mapping - type: enhancement - issues: [] - - pr: 125595 - summary: Esql - Fix lucene push down behavior when a range contains nanos and millis - area: ES|QL - type: bug - issues: [] - - pr: 125599 - summary: Allow zero for `rescore_vector.oversample` to indicate by-passing oversample and rescoring - area: Vector Search - type: enhancement - issues: [] - - pr: 125606 - summary: Rename deprecation index template - area: Infra/Logging - type: breaking - issues: - - 125445 - breaking: - area: Logging - title: Rename deprecation index template - details: "The deprecation datastream contains log entries for deprecations that occured while Elasticsearch is running. The deprecation log entries had a mismatch in their dataset name. In order to avoid changing the dataset name in the existing datastream, a new datastream now exists." - impact: "If querying for deprecations previously using the `.logs-deprecation.elasticsearch-default` datastream, you should now use the `.logs-elasticsearch.deprecation-default` datastream." - notable: false - essSettingChange: false - - pr: 125636 - summary: Make `numberOfChannels` consistent with layout map by removing duplicated `ChannelSet` - area: ES|QL - type: bug - issues: [] - - pr: 125650 - summary: Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` - area: Search - type: bug - issues: - - 125483 - - pr: 125659 - summary: Non existing synonyms sets do not fail shard recovery for indices - area: Analysis - type: bug - issues: - - 125603 - - pr: 125666 - summary: Minor-Fixes Support 7x segments as archive in 8x / 9x - area: Search - type: bug - issues: [] - - pr: 125709 - summary: Store arrays offsets for unsigned long fields natively with synthetic source - area: Mapping - type: enhancement - issues: [] - - pr: 125716 - summary: Return appropriate error on null dims update instead of npe - area: Vector Search - type: bug - issues: [] diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index dd3995ddea1ae..77741ce12dd2a 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -53,10 +53,7 @@ For more information, check [PR #124669](https://github.com/elastic/elasticsearc ### Features and enhancements [elasticsearch-900-features-enhancements] Allocation: -* Add `FailedShardEntry` info to shard-failed task source string [#125520](https://github.com/elastic/elasticsearch/pull/125520) (issue: [#125520](https://github.com/elastic/elasticsearch/pull/125520)) * Add a not-master state for desired balance [#116904](https://github.com/elastic/elasticsearch/pull/116904) -* Add cache support in `TransportGetAllocationStatsAction` [#124898](https://github.com/elastic/elasticsearch/pull/124898) (issue: [#124898](https://github.com/elastic/elasticsearch/pull/124898)) -* Introduce `AllocationBalancingRoundSummaryService` [#120957](https://github.com/elastic/elasticsearch/pull/120957) * Only publish desired balance gauges on master [#115383](https://github.com/elastic/elasticsearch/pull/115383) * Reset relocation/allocation failure counter on node join/shutdown [#119968](https://github.com/elastic/elasticsearch/pull/119968) @@ -75,7 +72,6 @@ CCS: * Resolve/cluster allows querying for cluster info only (no index expression required) [#119898](https://github.com/elastic/elasticsearch/pull/119898) CRUD: -* Enhance memory accounting for document expansion and introduce max document size limit [#123543](https://github.com/elastic/elasticsearch/pull/123543) * Metrics for indexing failures due to version conflicts [#119067](https://github.com/elastic/elasticsearch/pull/119067) * Remove INDEX_REFRESH_BLOCK after index becomes searchable [#120807](https://github.com/elastic/elasticsearch/pull/120807) * Suppress merge-on-recovery for older indices [#113462](https://github.com/elastic/elasticsearch/pull/113462) @@ -86,7 +82,6 @@ Cluster Coordination: Data streams: * Add action to create index from a source index [#118890](https://github.com/elastic/elasticsearch/pull/118890) * Add index and reindex request settings to speed up reindex [#119780](https://github.com/elastic/elasticsearch/pull/119780) -* Add index mode to get data stream API [#122486](https://github.com/elastic/elasticsearch/pull/122486) * Add rest endpoint for `create_from_source_index` [#119250](https://github.com/elastic/elasticsearch/pull/119250) * Add sanity check to `ReindexDatastreamIndexAction` [#120231](https://github.com/elastic/elasticsearch/pull/120231) * Adding a migration reindex cancel API [#118291](https://github.com/elastic/elasticsearch/pull/118291) @@ -101,9 +96,6 @@ Data streams: * Reindex data stream indices on different nodes [#125171](https://github.com/elastic/elasticsearch/pull/125171) * Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As A Warning [#120629](https://github.com/elastic/elasticsearch/pull/120629) * Retry ILM async action after reindexing data stream [#124149](https://github.com/elastic/elasticsearch/pull/124149) -* Run `TransportGetDataStreamLifecycleAction` on local node [#125214](https://github.com/elastic/elasticsearch/pull/125214) -* Run `TransportGetDataStreamOptionsAction` on local node [#125213](https://github.com/elastic/elasticsearch/pull/125213) -* Run `TransportGetDataStreamsAction` on local node [#122852](https://github.com/elastic/elasticsearch/pull/122852) * Set cause on create index request in create from action [#124363](https://github.com/elastic/elasticsearch/pull/124363) * Update data stream deprecations warnings to new format and filter searchable snapshots from response [#118562](https://github.com/elastic/elasticsearch/pull/118562) @@ -126,17 +118,11 @@ ES|QL: * Add a `PostAnalysisAware,` distribute verification [#119798](https://github.com/elastic/elasticsearch/pull/119798) * Add a standard deviation aggregating function: STD_DEV [#116531](https://github.com/elastic/elasticsearch/pull/116531) * Add cluster level reduction [#117731](https://github.com/elastic/elasticsearch/pull/117731) -* Add initial grammar and changes for FORK [#121948](https://github.com/elastic/elasticsearch/pull/121948) -* Add initial grammar and planning for RRF (snapshot) [#123396](https://github.com/elastic/elasticsearch/pull/123396) * Add nulls support to Categorize [#117655](https://github.com/elastic/elasticsearch/pull/117655) -* Allow partial results in ES|QL [#121942](https://github.com/elastic/elasticsearch/pull/121942) * Allow skip shards with `_tier` and `_index` in ES|QL [#123728](https://github.com/elastic/elasticsearch/pull/123728) * Async search responses have CCS metadata while searches are running [#117265](https://github.com/elastic/elasticsearch/pull/117265) -* Avoid `NamedWritable` in block serialization [#124394](https://github.com/elastic/elasticsearch/pull/124394) -* Calculate concurrent node limit [#124901](https://github.com/elastic/elasticsearch/pull/124901) * Check for early termination in Driver [#118188](https://github.com/elastic/elasticsearch/pull/118188) * Do not serialize `EsIndex` in plan [#119580](https://github.com/elastic/elasticsearch/pull/119580) -* Double parameter markers for identifiers [#122459](https://github.com/elastic/elasticsearch/pull/122459) * ESQL - Add Match function options [#120360](https://github.com/elastic/elasticsearch/pull/120360) * ESQL - Allow full text functions disjunctions for non-full text functions [#120291](https://github.com/elastic/elasticsearch/pull/120291) * ESQL - Remove restrictions for disjunctions in full text functions [#118544](https://github.com/elastic/elasticsearch/pull/118544) @@ -148,9 +134,7 @@ ES|QL: * ESQL: Enable async get to support formatting [#111104](https://github.com/elastic/elasticsearch/pull/111104) (issue: [#111104](https://github.com/elastic/elasticsearch/pull/111104)) * ESQL: Enterprise license enforcement for CCS [#118102](https://github.com/elastic/elasticsearch/pull/118102) * ES|QL - Add scoring for full text functions disjunctions [#121793](https://github.com/elastic/elasticsearch/pull/121793) -* ES|QL slow log [#124094](https://github.com/elastic/elasticsearch/pull/124094) * ES|QL: Partial result on demand for async queries [#118122](https://github.com/elastic/elasticsearch/pull/118122) -* ES|QL: Support `::date` in inline cast [#123460](https://github.com/elastic/elasticsearch/pull/123460) (issue: [#123460](https://github.com/elastic/elasticsearch/pull/123460)) * Enable KQL function as a tech preview [#119730](https://github.com/elastic/elasticsearch/pull/119730) * Enable LOOKUP JOIN in non-snapshot builds [#121193](https://github.com/elastic/elasticsearch/pull/121193) (issue: [#121193](https://github.com/elastic/elasticsearch/pull/121193)) * Enable node-level reduction by default [#119621](https://github.com/elastic/elasticsearch/pull/119621) @@ -165,50 +149,32 @@ ES|QL: * Expand type compatibility for match function and operator [#117555](https://github.com/elastic/elasticsearch/pull/117555) * Extend `TranslationAware` to all pushable expressions [#120192](https://github.com/elastic/elasticsearch/pull/120192) * Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: [#123290](https://github.com/elastic/elasticsearch/pull/123290)) -* Fix sorting when `aggregate_metric_double` present [#125191](https://github.com/elastic/elasticsearch/pull/125191) * Hash functions [#118938](https://github.com/elastic/elasticsearch/pull/118938) * Implement a `MetricsAware` interface [#121074](https://github.com/elastic/elasticsearch/pull/121074) -* Implement runtime skip_unavailable=true [#121240](https://github.com/elastic/elasticsearch/pull/121240) -* Include failures in partial response [#124929](https://github.com/elastic/elasticsearch/pull/124929) -* Infer the score mode to use from the Lucene collector [#125930](https://github.com/elastic/elasticsearch/pull/125930) * Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) -* Introduce `allow_partial_results` setting in ES|QL [#122890](https://github.com/elastic/elasticsearch/pull/122890) -* Introduce a pre-mapping logical plan processing step [#121260](https://github.com/elastic/elasticsearch/pull/121260) -* Keep ordinals in conversion functions [#125357](https://github.com/elastic/elasticsearch/pull/125357) * LOOKUP JOIN using field-caps for field mapping [#117246](https://github.com/elastic/elasticsearch/pull/117246) * Lookup join on multiple join fields not yet supported [#118858](https://github.com/elastic/elasticsearch/pull/118858) * Move scoring in ES|QL out of snapshot [#120354](https://github.com/elastic/elasticsearch/pull/120354) * Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` [#119889](https://github.com/elastic/elasticsearch/pull/119889) -* Pragma to load from stored fields [#122891](https://github.com/elastic/elasticsearch/pull/122891) * Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: [#123381](https://github.com/elastic/elasticsearch/pull/123381)) * Push down filter passed lookup join [#118410](https://github.com/elastic/elasticsearch/pull/118410) -* Remove page alignment in exchange sink [#124610](https://github.com/elastic/elasticsearch/pull/124610) -* Render `aggregate_metric_double` [#122660](https://github.com/elastic/elasticsearch/pull/122660) -* Report `original_types` [#124913](https://github.com/elastic/elasticsearch/pull/124913) -* Report failures on partial results [#124823](https://github.com/elastic/elasticsearch/pull/124823) * Resume Driver on cancelled or early finished [#120020](https://github.com/elastic/elasticsearch/pull/120020) -* Retry ES|QL node requests on shard level failures [#120774](https://github.com/elastic/elasticsearch/pull/120774) * Reuse child `outputSet` inside the plan where possible [#124611](https://github.com/elastic/elasticsearch/pull/124611) * Rewrite TO_UPPER/TO_LOWER comparisons [#118870](https://github.com/elastic/elasticsearch/pull/118870) (issue: [#118870](https://github.com/elastic/elasticsearch/pull/118870)) * ST_EXTENT aggregation [#117451](https://github.com/elastic/elasticsearch/pull/117451) (issue: [#117451](https://github.com/elastic/elasticsearch/pull/117451)) * ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape [#118802](https://github.com/elastic/elasticsearch/pull/118802) * Smarter field caps with subscribable listener [#116755](https://github.com/elastic/elasticsearch/pull/116755) * Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions [#116964](https://github.com/elastic/elasticsearch/pull/116964) (issue: [#116964](https://github.com/elastic/elasticsearch/pull/116964)) -* Support partial results in CCS in ES|QL [#122708](https://github.com/elastic/elasticsearch/pull/122708) * Support partial sort fields in TopN pushdown [#116043](https://github.com/elastic/elasticsearch/pull/116043) (issue: [#116043](https://github.com/elastic/elasticsearch/pull/116043)) * Support some stats on aggregate_metric_double [#120343](https://github.com/elastic/elasticsearch/pull/120343) (issue: [#120343](https://github.com/elastic/elasticsearch/pull/120343)) -* Support subset of metrics in aggregate metric double [#121805](https://github.com/elastic/elasticsearch/pull/121805) -* Take double parameter markers for identifiers out of snapshot [#125690](https://github.com/elastic/elasticsearch/pull/125690) * Take named parameters for identifier and pattern out of snapshot [#121850](https://github.com/elastic/elasticsearch/pull/121850) * Term query for ES|QL [#117359](https://github.com/elastic/elasticsearch/pull/117359) * Update grammar to rely on `indexPattern` instead of identifier in join target [#120494](https://github.com/elastic/elasticsearch/pull/120494) -* `ToAggregateMetricDouble` function [#124595](https://github.com/elastic/elasticsearch/pull/124595) * `_score` should not be a reserved attribute in ES|QL [#118435](https://github.com/elastic/elasticsearch/pull/118435) (issue: [#118435](https://github.com/elastic/elasticsearch/pull/118435)) Engine: * Defer unpromotable shard refreshes until index refresh blocks are cleared [#120642](https://github.com/elastic/elasticsearch/pull/120642) * POC mark read-only [#119743](https://github.com/elastic/elasticsearch/pull/119743) -* Threadpool merge scheduler [#120869](https://github.com/elastic/elasticsearch/pull/120869) Experiences: * Integrate IBM watsonx to Inference API for re-ranking task [#117176](https://github.com/elastic/elasticsearch/pull/117176) @@ -222,7 +188,6 @@ Geo: * Optimize indexing points with index and doc values set to true [#120271](https://github.com/elastic/elasticsearch/pull/120271) Health: -* Add health indicator impact to `HealthPeriodicLogger` [#122390](https://github.com/elastic/elasticsearch/pull/122390) * Increase `replica_unassigned_buffer_time` default from 3s to 5s [#112834](https://github.com/elastic/elasticsearch/pull/112834) Highlighting: @@ -230,20 +195,13 @@ Highlighting: ILM+SLM: * Add a `replicate_for` option to the ILM `searchable_snapshot` action [#119003](https://github.com/elastic/elasticsearch/pull/119003) -* Improve SLM Health Indicator to cover missing snapshot [#121370](https://github.com/elastic/elasticsearch/pull/121370) -* Optimize usage calculation in ILM policies retrieval API [#106953](https://github.com/elastic/elasticsearch/pull/106953) (issue: [#106953](https://github.com/elastic/elasticsearch/pull/106953)) -* Process ILM cluster state updates on another thread [#123712](https://github.com/elastic/elasticsearch/pull/123712) -* Run `TransportExplainLifecycleAction` on local node [#122885](https://github.com/elastic/elasticsearch/pull/122885) -* Truncate `step_info` and error reason in ILM execution state and history [#125054](https://github.com/elastic/elasticsearch/pull/125054) (issue: [#125054](https://github.com/elastic/elasticsearch/pull/125054)) Indices APIs: * Add `remove_index_block` arg to `_create_from` api [#120548](https://github.com/elastic/elasticsearch/pull/120548) -* Avoid creating known_fields for every check in Alias [#124690](https://github.com/elastic/elasticsearch/pull/124690) * Remove index blocks by default in `create_from` [#120643](https://github.com/elastic/elasticsearch/pull/120643) * Run `TransportGetComponentTemplateAction` on local node [#116868](https://github.com/elastic/elasticsearch/pull/116868) * Run `TransportGetComposableIndexTemplate` on local node [#119830](https://github.com/elastic/elasticsearch/pull/119830) * Run `TransportGetIndexTemplateAction` on local node [#119837](https://github.com/elastic/elasticsearch/pull/119837) -* Run `TransportGetMappingsAction` on local node [#122921](https://github.com/elastic/elasticsearch/pull/122921) * introduce new categories for deprecated resources in deprecation API [#120505](https://github.com/elastic/elasticsearch/pull/120505) Inference: @@ -251,7 +209,6 @@ Inference: * Remove Elastic Inference Service feature flag and deprecated setting [#120842](https://github.com/elastic/elasticsearch/pull/120842) * Update sparse text embeddings API route for Inference Service [#118025](https://github.com/elastic/elasticsearch/pull/118025) * [Elastic Inference Service] Add ElasticInferenceService Unified ChatCompletions Integration [#118871](https://github.com/elastic/elasticsearch/pull/118871) -* [Inference API] Rename `model_id` prop to model in EIS sparse inference request body [#122272](https://github.com/elastic/elasticsearch/pull/122272) Infra/CLI: * Ignore _JAVA_OPTIONS [#124843](https://github.com/elastic/elasticsearch/pull/124843) @@ -264,11 +221,9 @@ Infra/Core: * Add support for specifying reindexing script for system index migration [#119001](https://github.com/elastic/elasticsearch/pull/119001) * Bump major version for feature migration system indices [#117243](https://github.com/elastic/elasticsearch/pull/117243) * Change default Docker image to be based on UBI minimal instead of Ubuntu [#116739](https://github.com/elastic/elasticsearch/pull/116739) -* Give Kibana user 'all' permissions for .entity_analytics.* indices [#123588](https://github.com/elastic/elasticsearch/pull/123588) * Improve size limiting string message [#122427](https://github.com/elastic/elasticsearch/pull/122427) * Infrastructure for assuming cluster features in the next major version [#118143](https://github.com/elastic/elasticsearch/pull/118143) * Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#124865](https://github.com/elastic/elasticsearch/pull/124865) -* Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. [#125117](https://github.com/elastic/elasticsearch/pull/125117) * Update ASM 9.7 -> 9.7.1 to support JDK 24 [#118094](https://github.com/elastic/elasticsearch/pull/118094) Infra/Metrics: @@ -283,7 +238,6 @@ Infra/Scripting: * Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts [#114002](https://github.com/elastic/elasticsearch/pull/114002) Infra/Settings: -* Allow passing several reserved state chunks in single process call [#124574](https://github.com/elastic/elasticsearch/pull/124574) * Introduce `IndexSettingDeprecatedInV8AndRemovedInV9` Setting property [#120334](https://github.com/elastic/elasticsearch/pull/120334) * Run `TransportClusterGetSettingsAction` on local node [#119831](https://github.com/elastic/elasticsearch/pull/119831) @@ -295,7 +249,6 @@ Ingest Node: * Optimize some per-document hot paths in the geoip processor [#120824](https://github.com/elastic/elasticsearch/pull/120824) * Returning ignored fields in the simulate ingest API [#117214](https://github.com/elastic/elasticsearch/pull/117214) * Run `GetPipelineTransportAction` on local node [#120445](https://github.com/elastic/elasticsearch/pull/120445) -* Run `TransportEnrichStatsAction` on local node [#121256](https://github.com/elastic/elasticsearch/pull/121256) * Run `TransportGetEnrichPolicyAction` on local node [#121124](https://github.com/elastic/elasticsearch/pull/121124) * Run template simulation actions on local node [#120038](https://github.com/elastic/elasticsearch/pull/120038) @@ -312,56 +265,30 @@ Machine Learning: * Add DeBERTa-V2/V3 tokenizer [#111852](https://github.com/elastic/elasticsearch/pull/111852) * Add Inference Unified API for chat completions for OpenAI [#117589](https://github.com/elastic/elasticsearch/pull/117589) * Add Jina AI API to do inference for Embedding and Rerank models [#118652](https://github.com/elastic/elasticsearch/pull/118652) -* Add `ModelRegistryMetadata` to Cluster State [#121106](https://github.com/elastic/elasticsearch/pull/121106) * Add enterprise license check for Inference API actions [#119893](https://github.com/elastic/elasticsearch/pull/119893) * Adding chunking settings to `IbmWatsonxService` [#114914](https://github.com/elastic/elasticsearch/pull/114914) -* Adding common rerank options to Perform Inference API [#125239](https://github.com/elastic/elasticsearch/pull/125239) (issue: [#125239](https://github.com/elastic/elasticsearch/pull/125239)) * Adding default endpoint for Elastic Rerank [#117939](https://github.com/elastic/elasticsearch/pull/117939) -* Adding elser default endpoint for EIS [#122066](https://github.com/elastic/elasticsearch/pull/122066) * Adding endpoint creation validation for all task types to remaining services [#115020](https://github.com/elastic/elasticsearch/pull/115020) -* Adding endpoint creation validation to `ElasticInferenceService` [#117642](https://github.com/elastic/elasticsearch/pull/117642) -* Adding integration for VoyageAI embeddings and rerank models [#122134](https://github.com/elastic/elasticsearch/pull/122134) -* Adding support for binary embedding type to Cohere service embedding type [#120751](https://github.com/elastic/elasticsearch/pull/120751) -* Adding support for specifying embedding type to Jina AI service settings [#121548](https://github.com/elastic/elasticsearch/pull/121548) * Automatically rollover legacy .ml-anomalies indices [#120913](https://github.com/elastic/elasticsearch/pull/120913) * Automatically rollover legacy ml indices [#120405](https://github.com/elastic/elasticsearch/pull/120405) * Change the auditor to write via an alias [#120064](https://github.com/elastic/elasticsearch/pull/120064) * Check for presence of error object when validating streaming responses from integrations in the inference API [#118375](https://github.com/elastic/elasticsearch/pull/118375) * Check if the anomaly results index has been rolled over [#125404](https://github.com/elastic/elasticsearch/pull/125404) -* ES|QL `change_point` processing command [#120998](https://github.com/elastic/elasticsearch/pull/120998) * ES|QL categorize with multiple groupings [#118173](https://github.com/elastic/elasticsearch/pull/118173) -* Expose `input_type` option at root level for `text_embedding` task type in Perform Inference API [#122638](https://github.com/elastic/elasticsearch/pull/122638) (issue: [#122638](https://github.com/elastic/elasticsearch/pull/122638)) * Ignore failures from renormalizing buckets in read-only index [#118674](https://github.com/elastic/elasticsearch/pull/118674) * Inference duration and error metrics [#115876](https://github.com/elastic/elasticsearch/pull/115876) -* Integrate with `DeepSeek` API [#122218](https://github.com/elastic/elasticsearch/pull/122218) -* Limit the number of chunks for semantic text to prevent high memory usage [#123150](https://github.com/elastic/elasticsearch/pull/123150) * Migrate stream to core error parsing [#120722](https://github.com/elastic/elasticsearch/pull/120722) * Remove all mentions of eis and gateway and deprecate flags that do [#116692](https://github.com/elastic/elasticsearch/pull/116692) * Remove deprecated sort from reindex operation within dataframe analytics procedure [#117606](https://github.com/elastic/elasticsearch/pull/117606) * Retry on `ClusterBlockException` on transform destination index [#118194](https://github.com/elastic/elasticsearch/pull/118194) * Support mTLS for the Elastic Inference Service integration inside the inference API [#119679](https://github.com/elastic/elasticsearch/pull/119679) -* Upgrade AWS v2 SDK to 2.30.38 [#124738](https://github.com/elastic/elasticsearch/pull/124738) * [Inference API] Add node-local rate limiting for the inference API [#120400](https://github.com/elastic/elasticsearch/pull/120400) -* [Inference API] Propagate product use case http header to EIS [#124025](https://github.com/elastic/elasticsearch/pull/124025) * [Inference API] fix spell words: covertToString to convertToString [#119922](https://github.com/elastic/elasticsearch/pull/119922) Mapping: * Add Optional Source Filtering to Source Loaders [#113827](https://github.com/elastic/elasticsearch/pull/113827) * Add option to store `sparse_vector` outside `_source` [#117917](https://github.com/elastic/elasticsearch/pull/117917) -* Enable synthetic recovery source by default when synthetic source is enabled. Using synthetic recovery source significantly improves indexing performance compared to regular recovery source. [#122615](https://github.com/elastic/elasticsearch/pull/122615) (issue: [#122615](https://github.com/elastic/elasticsearch/pull/122615)) -* Enable the use of nested field type with index.mode=time_series [#122224](https://github.com/elastic/elasticsearch/pull/122224) (issue: [#122224](https://github.com/elastic/elasticsearch/pull/122224)) -* Improved error message when index field type is invalid [#122860](https://github.com/elastic/elasticsearch/pull/122860) -* Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields [#119546](https://github.com/elastic/elasticsearch/pull/119546) * Release semantic_text as a GA feature [#124669](https://github.com/elastic/elasticsearch/pull/124669) -* Store arrays offsets for boolean fields natively with synthetic source [#125529](https://github.com/elastic/elasticsearch/pull/125529) -* Store arrays offsets for ip fields natively with synthetic source [#122999](https://github.com/elastic/elasticsearch/pull/122999) -* Store arrays offsets for keyword fields natively with synthetic source instead of falling back to ignored source. [#113757](https://github.com/elastic/elasticsearch/pull/113757) -* Store arrays offsets for numeric fields natively with synthetic source [#124594](https://github.com/elastic/elasticsearch/pull/124594) -* Store arrays offsets for unsigned long fields natively with synthetic source [#125709](https://github.com/elastic/elasticsearch/pull/125709) -* Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` [#124927](https://github.com/elastic/elasticsearch/pull/124927) -* Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float` fields [#122637](https://github.com/elastic/elasticsearch/pull/122637) -* Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields [#124050](https://github.com/elastic/elasticsearch/pull/124050) -* Use `FallbackSyntheticSourceBlockLoader` for number fields [#122280](https://github.com/elastic/elasticsearch/pull/122280) Network: * Allow http unsafe buffers by default [#116115](https://github.com/elastic/elasticsearch/pull/116115) @@ -374,7 +301,6 @@ Packaging: Ranking: * Add a generic `rescorer` retriever based on the search request's rescore functionality [#118585](https://github.com/elastic/elasticsearch/pull/118585) (issue: [#118585](https://github.com/elastic/elasticsearch/pull/118585)) -* Leverage scorer supplier in `QueryFeatureExtractor` [#125259](https://github.com/elastic/elasticsearch/pull/125259) * Set default reranker for text similarity reranker to Elastic reranker [#120551](https://github.com/elastic/elasticsearch/pull/120551) Recovery: @@ -386,10 +312,8 @@ Reindex: Relevance: * Add Multi-Field Support for Semantic Text Fields [#120128](https://github.com/elastic/elasticsearch/pull/120128) -* Skip semantic_text embedding generation when no content is provided. [#123763](https://github.com/elastic/elasticsearch/pull/123763) Search: -* Account for the `SearchHit` source in circuit breaker [#121920](https://github.com/elastic/elasticsearch/pull/121920) (issue: [#121920](https://github.com/elastic/elasticsearch/pull/121920)) * Add match support for `semantic_text` fields [#117839](https://github.com/elastic/elasticsearch/pull/117839) * Add support for `sparse_vector` queries against `semantic_text` fields [#118617](https://github.com/elastic/elasticsearch/pull/118617) * Add support for knn vector queries on `semantic_text` fields [#119011](https://github.com/elastic/elasticsearch/pull/119011) @@ -397,11 +321,8 @@ Search: * Adding linear retriever to support weighted sums of sub-retrievers [#120222](https://github.com/elastic/elasticsearch/pull/120222) * Address and remove any references of RestApiVersion version 7 [#117572](https://github.com/elastic/elasticsearch/pull/117572) * Feat: add a user-configurable timeout parameter to the `_resolve/cluster` API [#120542](https://github.com/elastic/elasticsearch/pull/120542) -* Introduce batched query execution and data-node side reduce [#121885](https://github.com/elastic/elasticsearch/pull/121885) * Make semantic text part of the text family [#119792](https://github.com/elastic/elasticsearch/pull/119792) * Only aggregations require at least one shard request [#115314](https://github.com/elastic/elasticsearch/pull/115314) -* Optimize memory usage in `ShardBulkInferenceActionFilter` [#124313](https://github.com/elastic/elasticsearch/pull/124313) -* Optionally allow text similarity reranking to fail [#121784](https://github.com/elastic/elasticsearch/pull/121784) * Prevent data nodes from sending stack traces to coordinator when `error_trace=false` [#118266](https://github.com/elastic/elasticsearch/pull/118266) * Propagate status codes from shard failures appropriately [#118016](https://github.com/elastic/elasticsearch/pull/118016) (issue: [#118016](https://github.com/elastic/elasticsearch/pull/118016)) * Upgrade to Lucene 10 [#114741](https://github.com/elastic/elasticsearch/pull/114741) @@ -413,19 +334,11 @@ Security: Snapshot/Restore: * Add IMDSv2 support to `repository-s3` [#117748](https://github.com/elastic/elasticsearch/pull/117748) (issue: [#117748](https://github.com/elastic/elasticsearch/pull/117748)) * Expose operation and request counts separately in repository stats [#117530](https://github.com/elastic/elasticsearch/pull/117530) (issue: [#117530](https://github.com/elastic/elasticsearch/pull/117530)) -* GCS blob store: add `OperationPurpose/Operation` stats counters [#122991](https://github.com/elastic/elasticsearch/pull/122991) * Retry `S3BlobContainer#getRegister` on all exceptions [#114813](https://github.com/elastic/elasticsearch/pull/114813) * Retry internally when CAS upload is throttled [GCS] [#120250](https://github.com/elastic/elasticsearch/pull/120250) (issue: [#120250](https://github.com/elastic/elasticsearch/pull/120250)) -* Retry when the server can't be resolved (Google Cloud Storage) [#123852](https://github.com/elastic/elasticsearch/pull/123852) * Track shard snapshot progress during node shutdown [#112567](https://github.com/elastic/elasticsearch/pull/112567) * Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) -Stats: -* Run XPack usage actions on local node [#122933](https://github.com/elastic/elasticsearch/pull/122933) - -Store: -* Abort pending deletion on `IndicesService` close [#123569](https://github.com/elastic/elasticsearch/pull/123569) - Suggesters: * Extensible Completion Postings Formats [#111494](https://github.com/elastic/elasticsearch/pull/111494) @@ -440,29 +353,18 @@ Transform: * [Deprecation] Add `transform_ids` to outdated index [#120821](https://github.com/elastic/elasticsearch/pull/120821) Vector Search: -* Add bit vector support to semantic text [#123187](https://github.com/elastic/elasticsearch/pull/123187) * Add new experimental `rank_vectors` mapping for late-interaction second order ranking [#118804](https://github.com/elastic/elasticsearch/pull/118804) -* Add panama implementations of byte-bit and float-bit script operations [#124722](https://github.com/elastic/elasticsearch/pull/124722) (issue: [#124722](https://github.com/elastic/elasticsearch/pull/124722)) -* Adds implementations of dotProduct and cosineSimilarity painless methods to operate on float vectors for byte fields [#122381](https://github.com/elastic/elasticsearch/pull/122381) (issue: [#122381](https://github.com/elastic/elasticsearch/pull/122381)) -* Allow zero for `rescore_vector.oversample` to indicate by-passing oversample and rescoring [#125599](https://github.com/elastic/elasticsearch/pull/125599) * Even better(er) binary quantization [#117994](https://github.com/elastic/elasticsearch/pull/117994) * KNN vector rescoring for quantized vectors [#116663](https://github.com/elastic/elasticsearch/pull/116663) * Mark bbq indices as GA and add rolling upgrade integration tests [#121105](https://github.com/elastic/elasticsearch/pull/121105) -* New `vector_rescore` parameter as a quantized index type option [#124581](https://github.com/elastic/elasticsearch/pull/124581) * Speed up bit compared with floats or bytes script operations [#117199](https://github.com/elastic/elasticsearch/pull/117199) -Watcher: -* Run `TransportGetWatcherSettingsAction` on local node [#122857](https://github.com/elastic/elasticsearch/pull/122857) - ### Fixes [elasticsearch-900-fixes] Aggregations: * Aggs: Let terms run in global ords mode no match [#124782](https://github.com/elastic/elasticsearch/pull/124782) * Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations [#116174](https://github.com/elastic/elasticsearch/pull/116174) (issue: [#116174](https://github.com/elastic/elasticsearch/pull/116174)) -Allocation: -* `DesiredBalanceReconciler` always returns `AllocationStats` [#122458](https://github.com/elastic/elasticsearch/pull/122458) - Analysis: * Adjust exception thrown when unable to load hunspell dict [#123743](https://github.com/elastic/elasticsearch/pull/123743) * Analyze API to return 400 for wrong custom analyzer [#121568](https://github.com/elastic/elasticsearch/pull/121568) (issue: [#121568](https://github.com/elastic/elasticsearch/pull/121568)) @@ -478,9 +380,6 @@ CRUD: * Preserve thread context when waiting for segment generation in RTG [#114623](https://github.com/elastic/elasticsearch/pull/114623) * Preserve thread context when waiting for segment generation in RTG [#117148](https://github.com/elastic/elasticsearch/pull/117148) -Cluster Coordination: -* Disable logging in `ClusterFormationFailureHelper` on shutdown [#125244](https://github.com/elastic/elasticsearch/pull/125244) (issue: [#125244](https://github.com/elastic/elasticsearch/pull/125244)) - Data streams: * Avoid updating settings version in `MetadataMigrateToDataStreamService` when settings have not changed [#118704](https://github.com/elastic/elasticsearch/pull/118704) * Block-writes cannot be added after read-only [#119007](https://github.com/elastic/elasticsearch/pull/119007) (issue: [#119007](https://github.com/elastic/elasticsearch/pull/119007)) @@ -491,16 +390,10 @@ Data streams: * Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover [#122905](https://github.com/elastic/elasticsearch/pull/122905) * `ReindexDataStreamIndex` bug in assertion caused by reference equality [#121325](https://github.com/elastic/elasticsearch/pull/121325) -Distributed: -* Pass `IndexReshardingMetadata` over the wire [#124841](https://github.com/elastic/elasticsearch/pull/124841) - Downsampling: * Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` [#121727](https://github.com/elastic/elasticsearch/pull/121727) (issues: [#121727](https://github.com/elastic/elasticsearch/pull/121727), [#121727](https://github.com/elastic/elasticsearch/pull/121727)) * Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. [#124451](https://github.com/elastic/elasticsearch/pull/124451) -EQL: -* Fix EQL double invoking listener [#124918](https://github.com/elastic/elasticsearch/pull/124918) - ES|QL: * Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: [#122886](https://github.com/elastic/elasticsearch/pull/122886)) * Allow the data type of `null` in filters [#118324](https://github.com/elastic/elasticsearch/pull/118324) (issue: [#118324](https://github.com/elastic/elasticsearch/pull/118324)) @@ -510,7 +403,6 @@ ES|QL: * Drop null columns in text formats [#117643](https://github.com/elastic/elasticsearch/pull/117643) (issue: [#117643](https://github.com/elastic/elasticsearch/pull/117643)) * ESQL - date nanos range bug? [#125345](https://github.com/elastic/elasticsearch/pull/125345) (issue: [#125345](https://github.com/elastic/elasticsearch/pull/125345)) * ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate` [#124446](https://github.com/elastic/elasticsearch/pull/124446) (issue: [#124446](https://github.com/elastic/elasticsearch/pull/124446)) -* ESQL: Fix inconsistent results in using scaled_float field [#122586](https://github.com/elastic/elasticsearch/pull/122586) (issue: [#122586](https://github.com/elastic/elasticsearch/pull/122586)) * ESQL: Remove estimated row size assertion [#122762](https://github.com/elastic/elasticsearch/pull/122762) (issue: [#122762](https://github.com/elastic/elasticsearch/pull/122762)) * ES|QL: Fix scoring for full text functions [#124540](https://github.com/elastic/elasticsearch/pull/124540) * Esql - Fix lucene push down behavior when a range contains nanos and millis [#125595](https://github.com/elastic/elasticsearch/pull/125595) @@ -524,22 +416,18 @@ ES|QL: * Fix attribute set equals [#118823](https://github.com/elastic/elasticsearch/pull/118823) * Fix double lookup failure on ESQL [#115616](https://github.com/elastic/elasticsearch/pull/115616) (issue: [#115616](https://github.com/elastic/elasticsearch/pull/115616)) * Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: [#123492](https://github.com/elastic/elasticsearch/pull/123492)) -* Fix functions emitting warnings with no source [#122821](https://github.com/elastic/elasticsearch/pull/122821) (issue: [#122821](https://github.com/elastic/elasticsearch/pull/122821)) * Fix queries with document level security on lookup indexes [#120617](https://github.com/elastic/elasticsearch/pull/120617) (issue: [#120617](https://github.com/elastic/elasticsearch/pull/120617)) * Fix writing for LOOKUP status [#119296](https://github.com/elastic/elasticsearch/pull/119296) (issue: [#119296](https://github.com/elastic/elasticsearch/pull/119296)) * Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: [#122601](https://github.com/elastic/elasticsearch/pull/122601)) -* Improve error message for ( and [ [#124177](https://github.com/elastic/elasticsearch/pull/124177) (issue: [#124177](https://github.com/elastic/elasticsearch/pull/124177)) * Lazy collection copying during node transform [#124424](https://github.com/elastic/elasticsearch/pull/124424) * Limit memory usage of `fold` [#118602](https://github.com/elastic/elasticsearch/pull/118602) * Limit size of query [#117898](https://github.com/elastic/elasticsearch/pull/117898) * Make `numberOfChannels` consistent with layout map by removing duplicated `ChannelSet` [#125636](https://github.com/elastic/elasticsearch/pull/125636) * Reduce iteration complexity for plan traversal [#123427](https://github.com/elastic/elasticsearch/pull/123427) -* Remove duplicated nested commands [#123085](https://github.com/elastic/elasticsearch/pull/123085) * Remove redundant sorts from execution plan [#121156](https://github.com/elastic/elasticsearch/pull/121156) * Revert unwanted ES|QL lexer changes from PR #120354 [#120538](https://github.com/elastic/elasticsearch/pull/120538) * Revive inlinestats [#122257](https://github.com/elastic/elasticsearch/pull/122257) * Revive some more of inlinestats functionality [#123589](https://github.com/elastic/elasticsearch/pull/123589) -* TO_LOWER processes all values [#124676](https://github.com/elastic/elasticsearch/pull/124676) (issue: [#124676](https://github.com/elastic/elasticsearch/pull/124676)) * Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: [#124001](https://github.com/elastic/elasticsearch/pull/124001)) Engine: @@ -567,7 +455,6 @@ Infra/Core: * Fix system data streams to be restorable from a snapshot [#124651](https://github.com/elastic/elasticsearch/pull/124651) (issue: [#124651](https://github.com/elastic/elasticsearch/pull/124651)) * Have create index return a bad request on poor formatting [#123761](https://github.com/elastic/elasticsearch/pull/123761) * Include data streams when converting an existing resource to a system resource [#121392](https://github.com/elastic/elasticsearch/pull/121392) -* Reduce Data Loss in System Indices Migration [#121327](https://github.com/elastic/elasticsearch/pull/121327) * System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) * System data streams are not being upgraded in the feature migration API [#124884](https://github.com/elastic/elasticsearch/pull/124884) (issue: [#124884](https://github.com/elastic/elasticsearch/pull/124884)) * Wrap jackson exception on malformed json string [#114445](https://github.com/elastic/elasticsearch/pull/114445) (issue: [#114445](https://github.com/elastic/elasticsearch/pull/114445)) @@ -596,7 +483,6 @@ Ingest Node: * Fix geoip databases index access after system feature migration [#121196](https://github.com/elastic/elasticsearch/pull/121196) * Fix geoip databases index access after system feature migration (again) [#122938](https://github.com/elastic/elasticsearch/pull/122938) * Fix geoip databases index access after system feature migration (take 3) [#124604](https://github.com/elastic/elasticsearch/pull/124604) -* apm-data: Use representative count as event.success_count if available [#119995](https://github.com/elastic/elasticsearch/pull/119995) Logs: * Always check if index mode is logsdb [#116922](https://github.com/elastic/elasticsearch/pull/116922) @@ -611,13 +497,9 @@ Machine Learning: * Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) * Fixing bedrock event executor terminated cache issue [#118177](https://github.com/elastic/elasticsearch/pull/118177) (issue: [#118177](https://github.com/elastic/elasticsearch/pull/118177)) * Fixing bug setting index when parsing Google Vertex AI results [#117287](https://github.com/elastic/elasticsearch/pull/117287) -* Prevent get datafeeds stats API returning an error when local tasks are slow to stop [#125477](https://github.com/elastic/elasticsearch/pull/125477) (issue: [#125477](https://github.com/elastic/elasticsearch/pull/125477)) -* Provide model size statistics as soon as an anomaly detection job is opened [#124638](https://github.com/elastic/elasticsearch/pull/124638) (issue: [#124638](https://github.com/elastic/elasticsearch/pull/124638)) * Retry on streaming errors [#123076](https://github.com/elastic/elasticsearch/pull/123076) -* Return a Conflict status code if the model deployment is stopped by a user [#125204](https://github.com/elastic/elasticsearch/pull/125204) (issue: [#125204](https://github.com/elastic/elasticsearch/pull/125204)) * Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) * Set default similarity for Cohere model to cosine [#125370](https://github.com/elastic/elasticsearch/pull/125370) (issue: [#125370](https://github.com/elastic/elasticsearch/pull/125370)) -* Updates to allow using Cohere binary embedding response in semantic search queries [#121827](https://github.com/elastic/elasticsearch/pull/121827) * Updating Inference Update API documentation to have the correct PUT method [#121048](https://github.com/elastic/elasticsearch/pull/121048) * Wait for up to 2 seconds for yellow status before starting search [#115938](https://github.com/elastic/elasticsearch/pull/115938) (issues: [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938)) * [Inference API] Fix output stream ordering in `InferenceActionProxy` [#124225](https://github.com/elastic/elasticsearch/pull/124225) @@ -638,14 +520,9 @@ Network: Ranking: * Fix LTR query feature with phrases (and two-phase) queries [#125103](https://github.com/elastic/elasticsearch/pull/125103) -* Restore `TextSimilarityRankBuilder` XContent output [#124564](https://github.com/elastic/elasticsearch/pull/124564) - -Relevance: -* Prevent Query Rule Creation with Invalid Numeric Match Criteria [#122823](https://github.com/elastic/elasticsearch/pull/122823) Search: * Catch and handle disconnect exceptions in search [#115836](https://github.com/elastic/elasticsearch/pull/115836) -* Fix handling of auto expand replicas for stateless indices [#122365](https://github.com/elastic/elasticsearch/pull/122365) * Fix leak in `DfsQueryPhase` and introduce search disconnect stress test [#116060](https://github.com/elastic/elasticsearch/pull/116060) (issue: [#116060](https://github.com/elastic/elasticsearch/pull/116060)) * Fix/QueryBuilderBWCIT_muted_test [#117831](https://github.com/elastic/elasticsearch/pull/117831) * Handle long overflow in dates [#124048](https://github.com/elastic/elasticsearch/pull/124048) (issue: [#124048](https://github.com/elastic/elasticsearch/pull/124048)) @@ -669,7 +546,6 @@ Search: Snapshot/Restore: * Add undeclared Azure settings, modify test to exercise them [#118634](https://github.com/elastic/elasticsearch/pull/118634) * Fork post-snapshot-delete cleanup off master thread [#122731](https://github.com/elastic/elasticsearch/pull/122731) -* Limit number of suppressed S3 deletion errors [#123630](https://github.com/elastic/elasticsearch/pull/123630) (issue: [#123630](https://github.com/elastic/elasticsearch/pull/123630)) * Retry throttled snapshot deletions [#113237](https://github.com/elastic/elasticsearch/pull/113237) * This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. [#124931](https://github.com/elastic/elasticsearch/pull/124931) * Use the system index descriptor in the snapshot blob cache cleanup task [#120937](https://github.com/elastic/elasticsearch/pull/120937) (issue: [#120937](https://github.com/elastic/elasticsearch/pull/120937)) @@ -679,7 +555,6 @@ Store: Suggesters: * Return an empty suggestion when suggest phase times out [#122575](https://github.com/elastic/elasticsearch/pull/122575) (issue: [#122575](https://github.com/elastic/elasticsearch/pull/122575)) -* Support duplicate suggestions in completion field [#121324](https://github.com/elastic/elasticsearch/pull/121324) (issue: [#121324](https://github.com/elastic/elasticsearch/pull/121324)) Transform: * If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias [#122074](https://github.com/elastic/elasticsearch/pull/122074) (issue: [#122074](https://github.com/elastic/elasticsearch/pull/122074)) From 21b6e830091a9e142729960052f1ee8346cce4f9 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 14 Apr 2025 14:29:16 -0400 Subject: [PATCH 17/34] Fix issue links --- .../src/main/resources/templates/index.md | 2 +- docs/release-notes/index.md | 170 +++++++++--------- 2 files changed, 86 insertions(+), 86 deletions(-) diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md index dcb87b45c8dfb..bcebcf83c1fc8 100644 --- a/build-tools-internal/src/main/resources/templates/index.md +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -60,7 +60,7 @@ for (changeType in ['features-enhancements', 'fixes', 'regression']) { print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" if (change.issues != null && change.issues.empty == false) { print change.issues.size() == 1 ? " (issue: " : " (issues: " - print change.issues.collect { "[#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" }.join(", ") + print change.issues.collect { "[#${it}](https://github.com/elastic/elasticsearch/issues/${it})" }.join(", ") print ")" } print "\n" diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 77741ce12dd2a..e659feea03c9e 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -114,7 +114,7 @@ EQL: ES|QL: * Add ES|QL cross-cluster query telemetry collection [#119474](https://github.com/elastic/elasticsearch/pull/119474) -* Add a `LicenseAware` interface for licensed Nodes [#118931](https://github.com/elastic/elasticsearch/pull/118931) (issue: [#118931](https://github.com/elastic/elasticsearch/pull/118931)) +* Add a `LicenseAware` interface for licensed Nodes [#118931](https://github.com/elastic/elasticsearch/pull/118931) (issue: [#117405](https://github.com/elastic/elasticsearch/issues/117405)) * Add a `PostAnalysisAware,` distribute verification [#119798](https://github.com/elastic/elasticsearch/pull/119798) * Add a standard deviation aggregating function: STD_DEV [#116531](https://github.com/elastic/elasticsearch/pull/116531) * Add cluster level reduction [#117731](https://github.com/elastic/elasticsearch/pull/117731) @@ -128,27 +128,27 @@ ES|QL: * ESQL - Remove restrictions for disjunctions in full text functions [#118544](https://github.com/elastic/elasticsearch/pull/118544) * ESQL - enabling scoring with METADATA `_score` [#113120](https://github.com/elastic/elasticsearch/pull/113120) * ESQL Add esql hash function [#117989](https://github.com/elastic/elasticsearch/pull/117989) -* ESQL Support IN operator for Date nanos [#119772](https://github.com/elastic/elasticsearch/pull/119772) (issue: [#119772](https://github.com/elastic/elasticsearch/pull/119772)) -* ESQL: Align `RENAME` behavior with `EVAL` for sequential processing [#122250](https://github.com/elastic/elasticsearch/pull/122250) (issue: [#122250](https://github.com/elastic/elasticsearch/pull/122250)) +* ESQL Support IN operator for Date nanos [#119772](https://github.com/elastic/elasticsearch/pull/119772) (issue: [#118578](https://github.com/elastic/elasticsearch/issues/118578)) +* ESQL: Align `RENAME` behavior with `EVAL` for sequential processing [#122250](https://github.com/elastic/elasticsearch/pull/122250) (issue: [#121739](https://github.com/elastic/elasticsearch/issues/121739)) * ESQL: CATEGORIZE as a `BlockHash` [#114317](https://github.com/elastic/elasticsearch/pull/114317) -* ESQL: Enable async get to support formatting [#111104](https://github.com/elastic/elasticsearch/pull/111104) (issue: [#111104](https://github.com/elastic/elasticsearch/pull/111104)) +* ESQL: Enable async get to support formatting [#111104](https://github.com/elastic/elasticsearch/pull/111104) (issue: [#110926](https://github.com/elastic/elasticsearch/issues/110926)) * ESQL: Enterprise license enforcement for CCS [#118102](https://github.com/elastic/elasticsearch/pull/118102) * ES|QL - Add scoring for full text functions disjunctions [#121793](https://github.com/elastic/elasticsearch/pull/121793) * ES|QL: Partial result on demand for async queries [#118122](https://github.com/elastic/elasticsearch/pull/118122) * Enable KQL function as a tech preview [#119730](https://github.com/elastic/elasticsearch/pull/119730) -* Enable LOOKUP JOIN in non-snapshot builds [#121193](https://github.com/elastic/elasticsearch/pull/121193) (issue: [#121193](https://github.com/elastic/elasticsearch/pull/121193)) +* Enable LOOKUP JOIN in non-snapshot builds [#121193](https://github.com/elastic/elasticsearch/pull/121193) (issue: [#121185](https://github.com/elastic/elasticsearch/issues/121185)) * Enable node-level reduction by default [#119621](https://github.com/elastic/elasticsearch/pull/119621) * Enable physical plan verification [#118114](https://github.com/elastic/elasticsearch/pull/118114) * Ensure cluster string could be quoted [#120355](https://github.com/elastic/elasticsearch/pull/120355) -* Esql - Support date nanos in date extract function [#120727](https://github.com/elastic/elasticsearch/pull/120727) (issue: [#120727](https://github.com/elastic/elasticsearch/pull/120727)) -* Esql - support date nanos in date format function [#120143](https://github.com/elastic/elasticsearch/pull/120143) (issue: [#120143](https://github.com/elastic/elasticsearch/pull/120143)) -* Esql Support date nanos on date diff function [#120645](https://github.com/elastic/elasticsearch/pull/120645) (issue: [#120645](https://github.com/elastic/elasticsearch/pull/120645)) -* Esql bucket function for date nanos [#118474](https://github.com/elastic/elasticsearch/pull/118474) (issue: [#118474](https://github.com/elastic/elasticsearch/pull/118474)) -* Esql compare nanos and millis [#118027](https://github.com/elastic/elasticsearch/pull/118027) (issue: [#118027](https://github.com/elastic/elasticsearch/pull/118027)) -* Esql implicit casting for date nanos [#118697](https://github.com/elastic/elasticsearch/pull/118697) (issue: [#118697](https://github.com/elastic/elasticsearch/pull/118697)) +* Esql - Support date nanos in date extract function [#120727](https://github.com/elastic/elasticsearch/pull/120727) (issue: [#110000](https://github.com/elastic/elasticsearch/issues/110000)) +* Esql - support date nanos in date format function [#120143](https://github.com/elastic/elasticsearch/pull/120143) (issue: [#109994](https://github.com/elastic/elasticsearch/issues/109994)) +* Esql Support date nanos on date diff function [#120645](https://github.com/elastic/elasticsearch/pull/120645) (issue: [#109999](https://github.com/elastic/elasticsearch/issues/109999)) +* Esql bucket function for date nanos [#118474](https://github.com/elastic/elasticsearch/pull/118474) (issue: [#118031](https://github.com/elastic/elasticsearch/issues/118031)) +* Esql compare nanos and millis [#118027](https://github.com/elastic/elasticsearch/pull/118027) (issue: [#116281](https://github.com/elastic/elasticsearch/issues/116281)) +* Esql implicit casting for date nanos [#118697](https://github.com/elastic/elasticsearch/pull/118697) (issue: [#118476](https://github.com/elastic/elasticsearch/issues/118476)) * Expand type compatibility for match function and operator [#117555](https://github.com/elastic/elasticsearch/pull/117555) * Extend `TranslationAware` to all pushable expressions [#120192](https://github.com/elastic/elasticsearch/pull/120192) -* Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: [#123290](https://github.com/elastic/elasticsearch/pull/123290)) +* Fix Driver status iterations and `cpuTime` [#123290](https://github.com/elastic/elasticsearch/pull/123290) (issue: [#122967](https://github.com/elastic/elasticsearch/issues/122967)) * Hash functions [#118938](https://github.com/elastic/elasticsearch/pull/118938) * Implement a `MetricsAware` interface [#121074](https://github.com/elastic/elasticsearch/pull/121074) * Initial support for unmapped fields [#119886](https://github.com/elastic/elasticsearch/pull/119886) @@ -156,21 +156,21 @@ ES|QL: * Lookup join on multiple join fields not yet supported [#118858](https://github.com/elastic/elasticsearch/pull/118858) * Move scoring in ES|QL out of snapshot [#120354](https://github.com/elastic/elasticsearch/pull/120354) * Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` [#119889](https://github.com/elastic/elasticsearch/pull/119889) -* Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: [#123381](https://github.com/elastic/elasticsearch/pull/123381)) +* Push down `StartsWith` and `EndsWith` functions to Lucene [#123381](https://github.com/elastic/elasticsearch/pull/123381) (issue: [#123067](https://github.com/elastic/elasticsearch/issues/123067)) * Push down filter passed lookup join [#118410](https://github.com/elastic/elasticsearch/pull/118410) * Resume Driver on cancelled or early finished [#120020](https://github.com/elastic/elasticsearch/pull/120020) * Reuse child `outputSet` inside the plan where possible [#124611](https://github.com/elastic/elasticsearch/pull/124611) -* Rewrite TO_UPPER/TO_LOWER comparisons [#118870](https://github.com/elastic/elasticsearch/pull/118870) (issue: [#118870](https://github.com/elastic/elasticsearch/pull/118870)) -* ST_EXTENT aggregation [#117451](https://github.com/elastic/elasticsearch/pull/117451) (issue: [#117451](https://github.com/elastic/elasticsearch/pull/117451)) +* Rewrite TO_UPPER/TO_LOWER comparisons [#118870](https://github.com/elastic/elasticsearch/pull/118870) (issue: [#118304](https://github.com/elastic/elasticsearch/issues/118304)) +* ST_EXTENT aggregation [#117451](https://github.com/elastic/elasticsearch/pull/117451) (issue: [#104659](https://github.com/elastic/elasticsearch/issues/104659)) * ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape [#118802](https://github.com/elastic/elasticsearch/pull/118802) * Smarter field caps with subscribable listener [#116755](https://github.com/elastic/elasticsearch/pull/116755) -* Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions [#116964](https://github.com/elastic/elasticsearch/pull/116964) (issue: [#116964](https://github.com/elastic/elasticsearch/pull/116964)) -* Support partial sort fields in TopN pushdown [#116043](https://github.com/elastic/elasticsearch/pull/116043) (issue: [#116043](https://github.com/elastic/elasticsearch/pull/116043)) -* Support some stats on aggregate_metric_double [#120343](https://github.com/elastic/elasticsearch/pull/120343) (issue: [#120343](https://github.com/elastic/elasticsearch/pull/120343)) +* Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions [#116964](https://github.com/elastic/elasticsearch/pull/116964) (issue: [#104875](https://github.com/elastic/elasticsearch/issues/104875)) +* Support partial sort fields in TopN pushdown [#116043](https://github.com/elastic/elasticsearch/pull/116043) (issue: [#114515](https://github.com/elastic/elasticsearch/issues/114515)) +* Support some stats on aggregate_metric_double [#120343](https://github.com/elastic/elasticsearch/pull/120343) (issue: [#110649](https://github.com/elastic/elasticsearch/issues/110649)) * Take named parameters for identifier and pattern out of snapshot [#121850](https://github.com/elastic/elasticsearch/pull/121850) * Term query for ES|QL [#117359](https://github.com/elastic/elasticsearch/pull/117359) * Update grammar to rely on `indexPattern` instead of identifier in join target [#120494](https://github.com/elastic/elasticsearch/pull/120494) -* `_score` should not be a reserved attribute in ES|QL [#118435](https://github.com/elastic/elasticsearch/pull/118435) (issue: [#118435](https://github.com/elastic/elasticsearch/pull/118435)) +* `_score` should not be a reserved attribute in ES|QL [#118435](https://github.com/elastic/elasticsearch/pull/118435) (issue: [#118460](https://github.com/elastic/elasticsearch/issues/118460)) Engine: * Defer unpromotable shard refreshes until index refresh blocks are cleared [#120642](https://github.com/elastic/elasticsearch/pull/120642) @@ -242,7 +242,7 @@ Infra/Settings: * Run `TransportClusterGetSettingsAction` on local node [#119831](https://github.com/elastic/elasticsearch/pull/119831) Ingest Node: -* Allow setting the `type` in the reroute processor [#122409](https://github.com/elastic/elasticsearch/pull/122409) (issue: [#122409](https://github.com/elastic/elasticsearch/pull/122409)) +* Allow setting the `type` in the reroute processor [#122409](https://github.com/elastic/elasticsearch/pull/122409) (issue: [#121553](https://github.com/elastic/elasticsearch/issues/121553)) * Optimize `IngestCtxMap` construction [#120833](https://github.com/elastic/elasticsearch/pull/120833) * Optimize `IngestDocMetadata` `isAvailable` [#120753](https://github.com/elastic/elasticsearch/pull/120753) * Optimize `IngestDocument` `FieldPath` allocation [#120573](https://github.com/elastic/elasticsearch/pull/120573) @@ -258,7 +258,7 @@ License: Logs: * Add LogsDB option to route on sort fields [#116687](https://github.com/elastic/elasticsearch/pull/116687) * Add a new index setting to skip recovery source when synthetic source is enabled [#114618](https://github.com/elastic/elasticsearch/pull/114618) -* Configure index sorting through index settings for logsdb [#118968](https://github.com/elastic/elasticsearch/pull/118968) (issue: [#118968](https://github.com/elastic/elasticsearch/pull/118968)) +* Configure index sorting through index settings for logsdb [#118968](https://github.com/elastic/elasticsearch/pull/118968) (issue: [#118686](https://github.com/elastic/elasticsearch/issues/118686)) * Optimize loading mappings when determining synthetic source usage and whether host.name can be sorted on. [#120055](https://github.com/elastic/elasticsearch/pull/120055) Machine Learning: @@ -294,13 +294,13 @@ Network: * Allow http unsafe buffers by default [#116115](https://github.com/elastic/elasticsearch/pull/116115) * Http stream activity tracker and exceptions handling [#119564](https://github.com/elastic/elasticsearch/pull/119564) * Remove HTTP content copies [#117303](https://github.com/elastic/elasticsearch/pull/117303) -* `ConnectTransportException` returns retryable BAD_GATEWAY [#118681](https://github.com/elastic/elasticsearch/pull/118681) (issue: [#118681](https://github.com/elastic/elasticsearch/pull/118681)) +* `ConnectTransportException` returns retryable BAD_GATEWAY [#118681](https://github.com/elastic/elasticsearch/pull/118681) (issue: [#118320](https://github.com/elastic/elasticsearch/issues/118320)) Packaging: * Update bundled JDK to Java 24 [#125159](https://github.com/elastic/elasticsearch/pull/125159) Ranking: -* Add a generic `rescorer` retriever based on the search request's rescore functionality [#118585](https://github.com/elastic/elasticsearch/pull/118585) (issue: [#118585](https://github.com/elastic/elasticsearch/pull/118585)) +* Add a generic `rescorer` retriever based on the search request's rescore functionality [#118585](https://github.com/elastic/elasticsearch/pull/118585) (issue: [#118327](https://github.com/elastic/elasticsearch/issues/118327)) * Set default reranker for text similarity reranker to Elastic reranker [#120551](https://github.com/elastic/elasticsearch/pull/120551) Recovery: @@ -317,14 +317,14 @@ Search: * Add match support for `semantic_text` fields [#117839](https://github.com/elastic/elasticsearch/pull/117839) * Add support for `sparse_vector` queries against `semantic_text` fields [#118617](https://github.com/elastic/elasticsearch/pull/118617) * Add support for knn vector queries on `semantic_text` fields [#119011](https://github.com/elastic/elasticsearch/pull/119011) -* Added optional parameters to QSTR ES|QL function [#121787](https://github.com/elastic/elasticsearch/pull/121787) (issue: [#121787](https://github.com/elastic/elasticsearch/pull/121787)) +* Added optional parameters to QSTR ES|QL function [#121787](https://github.com/elastic/elasticsearch/pull/121787) (issue: [#120933](https://github.com/elastic/elasticsearch/issues/120933)) * Adding linear retriever to support weighted sums of sub-retrievers [#120222](https://github.com/elastic/elasticsearch/pull/120222) * Address and remove any references of RestApiVersion version 7 [#117572](https://github.com/elastic/elasticsearch/pull/117572) * Feat: add a user-configurable timeout parameter to the `_resolve/cluster` API [#120542](https://github.com/elastic/elasticsearch/pull/120542) * Make semantic text part of the text family [#119792](https://github.com/elastic/elasticsearch/pull/119792) * Only aggregations require at least one shard request [#115314](https://github.com/elastic/elasticsearch/pull/115314) * Prevent data nodes from sending stack traces to coordinator when `error_trace=false` [#118266](https://github.com/elastic/elasticsearch/pull/118266) -* Propagate status codes from shard failures appropriately [#118016](https://github.com/elastic/elasticsearch/pull/118016) (issue: [#118016](https://github.com/elastic/elasticsearch/pull/118016)) +* Propagate status codes from shard failures appropriately [#118016](https://github.com/elastic/elasticsearch/pull/118016) (issue: [#118482](https://github.com/elastic/elasticsearch/issues/118482)) * Upgrade to Lucene 10 [#114741](https://github.com/elastic/elasticsearch/pull/114741) * Upgrade to Lucene 10.1.0 [#119308](https://github.com/elastic/elasticsearch/pull/119308) @@ -332,10 +332,10 @@ Security: * Add refresh `.security` index call between security migrations [#114879](https://github.com/elastic/elasticsearch/pull/114879) Snapshot/Restore: -* Add IMDSv2 support to `repository-s3` [#117748](https://github.com/elastic/elasticsearch/pull/117748) (issue: [#117748](https://github.com/elastic/elasticsearch/pull/117748)) -* Expose operation and request counts separately in repository stats [#117530](https://github.com/elastic/elasticsearch/pull/117530) (issue: [#117530](https://github.com/elastic/elasticsearch/pull/117530)) +* Add IMDSv2 support to `repository-s3` [#117748](https://github.com/elastic/elasticsearch/pull/117748) (issue: [#105135](https://github.com/elastic/elasticsearch/issues/105135)) +* Expose operation and request counts separately in repository stats [#117530](https://github.com/elastic/elasticsearch/pull/117530) (issue: [#104443](https://github.com/elastic/elasticsearch/issues/104443)) * Retry `S3BlobContainer#getRegister` on all exceptions [#114813](https://github.com/elastic/elasticsearch/pull/114813) -* Retry internally when CAS upload is throttled [GCS] [#120250](https://github.com/elastic/elasticsearch/pull/120250) (issue: [#120250](https://github.com/elastic/elasticsearch/pull/120250)) +* Retry internally when CAS upload is throttled [GCS] [#120250](https://github.com/elastic/elasticsearch/pull/120250) (issue: [#116546](https://github.com/elastic/elasticsearch/issues/116546)) * Track shard snapshot progress during node shutdown [#112567](https://github.com/elastic/elasticsearch/pull/112567) * Upgrade AWS SDK to v1.12.746 [#122431](https://github.com/elastic/elasticsearch/pull/122431) @@ -363,12 +363,12 @@ Vector Search: Aggregations: * Aggs: Let terms run in global ords mode no match [#124782](https://github.com/elastic/elasticsearch/pull/124782) -* Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations [#116174](https://github.com/elastic/elasticsearch/pull/116174) (issue: [#116174](https://github.com/elastic/elasticsearch/pull/116174)) +* Handle with `illegalArgumentExceptions` negative values in HDR percentile aggregations [#116174](https://github.com/elastic/elasticsearch/pull/116174) (issue: [#115777](https://github.com/elastic/elasticsearch/issues/115777)) Analysis: * Adjust exception thrown when unable to load hunspell dict [#123743](https://github.com/elastic/elasticsearch/pull/123743) -* Analyze API to return 400 for wrong custom analyzer [#121568](https://github.com/elastic/elasticsearch/pull/121568) (issue: [#121568](https://github.com/elastic/elasticsearch/pull/121568)) -* Non existing synonyms sets do not fail shard recovery for indices [#125659](https://github.com/elastic/elasticsearch/pull/125659) (issue: [#125659](https://github.com/elastic/elasticsearch/pull/125659)) +* Analyze API to return 400 for wrong custom analyzer [#121568](https://github.com/elastic/elasticsearch/pull/121568) (issue: [#121443](https://github.com/elastic/elasticsearch/issues/121443)) +* Non existing synonyms sets do not fail shard recovery for indices [#125659](https://github.com/elastic/elasticsearch/pull/125659) (issue: [#125603](https://github.com/elastic/elasticsearch/issues/125603)) Authentication: * Fix NPE for missing Content Type header in OIDC Authenticator [#126191](https://github.com/elastic/elasticsearch/pull/126191) @@ -382,43 +382,43 @@ CRUD: Data streams: * Avoid updating settings version in `MetadataMigrateToDataStreamService` when settings have not changed [#118704](https://github.com/elastic/elasticsearch/pull/118704) -* Block-writes cannot be added after read-only [#119007](https://github.com/elastic/elasticsearch/pull/119007) (issue: [#119007](https://github.com/elastic/elasticsearch/pull/119007)) +* Block-writes cannot be added after read-only [#119007](https://github.com/elastic/elasticsearch/pull/119007) (issue: [#119002](https://github.com/elastic/elasticsearch/issues/119002)) * Ensure removal of index blocks does not leave key with null value [#122246](https://github.com/elastic/elasticsearch/pull/122246) * Fixes a invalid warning from being issued when restoring a system data stream from a snapshot. [#125881](https://github.com/elastic/elasticsearch/pull/125881) * Match dot prefix of migrated DS backing index with the source index [#120042](https://github.com/elastic/elasticsearch/pull/120042) -* Refresh source index before reindexing data stream index [#120752](https://github.com/elastic/elasticsearch/pull/120752) (issue: [#120752](https://github.com/elastic/elasticsearch/pull/120752)) +* Refresh source index before reindexing data stream index [#120752](https://github.com/elastic/elasticsearch/pull/120752) (issue: [#120314](https://github.com/elastic/elasticsearch/issues/120314)) * Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover [#122905](https://github.com/elastic/elasticsearch/pull/122905) * `ReindexDataStreamIndex` bug in assertion caused by reference equality [#121325](https://github.com/elastic/elasticsearch/pull/121325) Downsampling: -* Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` [#121727](https://github.com/elastic/elasticsearch/pull/121727) (issues: [#121727](https://github.com/elastic/elasticsearch/pull/121727), [#121727](https://github.com/elastic/elasticsearch/pull/121727)) +* Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` [#121727](https://github.com/elastic/elasticsearch/pull/121727) (issues: [#119696](https://github.com/elastic/elasticsearch/issues/119696), [#96076](https://github.com/elastic/elasticsearch/issues/96076)) * Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. [#124451](https://github.com/elastic/elasticsearch/pull/124451) ES|QL: -* Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: [#122886](https://github.com/elastic/elasticsearch/pull/122886)) -* Allow the data type of `null` in filters [#118324](https://github.com/elastic/elasticsearch/pull/118324) (issue: [#118324](https://github.com/elastic/elasticsearch/pull/118324)) +* Add support to VALUES aggregation for spatial types [#122886](https://github.com/elastic/elasticsearch/pull/122886) (issue: [#122413](https://github.com/elastic/elasticsearch/issues/122413)) +* Allow the data type of `null` in filters [#118324](https://github.com/elastic/elasticsearch/pull/118324) (issue: [#116351](https://github.com/elastic/elasticsearch/issues/116351)) * Avoid over collecting in Limit or Lucene Operator [#123296](https://github.com/elastic/elasticsearch/pull/123296) * Change the order of the optimization rules [#124335](https://github.com/elastic/elasticsearch/pull/124335) * Correct line and column numbers of missing named parameters [#120852](https://github.com/elastic/elasticsearch/pull/120852) -* Drop null columns in text formats [#117643](https://github.com/elastic/elasticsearch/pull/117643) (issue: [#117643](https://github.com/elastic/elasticsearch/pull/117643)) -* ESQL - date nanos range bug? [#125345](https://github.com/elastic/elasticsearch/pull/125345) (issue: [#125345](https://github.com/elastic/elasticsearch/pull/125345)) -* ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate` [#124446](https://github.com/elastic/elasticsearch/pull/124446) (issue: [#124446](https://github.com/elastic/elasticsearch/pull/124446)) -* ESQL: Remove estimated row size assertion [#122762](https://github.com/elastic/elasticsearch/pull/122762) (issue: [#122762](https://github.com/elastic/elasticsearch/pull/122762)) +* Drop null columns in text formats [#117643](https://github.com/elastic/elasticsearch/pull/117643) (issue: [#116848](https://github.com/elastic/elasticsearch/issues/116848)) +* ESQL - date nanos range bug? [#125345](https://github.com/elastic/elasticsearch/pull/125345) (issue: [#125439](https://github.com/elastic/elasticsearch/issues/125439)) +* ESQL: Fail in `AggregateFunction` when `LogicPlan` is not an `Aggregate` [#124446](https://github.com/elastic/elasticsearch/pull/124446) (issue: [#124311](https://github.com/elastic/elasticsearch/issues/124311)) +* ESQL: Remove estimated row size assertion [#122762](https://github.com/elastic/elasticsearch/pull/122762) (issue: [#121535](https://github.com/elastic/elasticsearch/issues/121535)) * ES|QL: Fix scoring for full text functions [#124540](https://github.com/elastic/elasticsearch/pull/124540) * Esql - Fix lucene push down behavior when a range contains nanos and millis [#125595](https://github.com/elastic/elasticsearch/pull/125595) * Fix ROUND() with unsigned longs throwing in some edge cases [#119536](https://github.com/elastic/elasticsearch/pull/119536) -* Fix TDigestState.read CB leaks [#114303](https://github.com/elastic/elasticsearch/pull/114303) (issue: [#114303](https://github.com/elastic/elasticsearch/pull/114303)) -* Fix TopN row size estimate [#119476](https://github.com/elastic/elasticsearch/pull/119476) (issue: [#119476](https://github.com/elastic/elasticsearch/pull/119476)) -* Fix `AbstractShapeGeometryFieldMapperTests` [#119265](https://github.com/elastic/elasticsearch/pull/119265) (issue: [#119265](https://github.com/elastic/elasticsearch/pull/119265)) -* Fix `ReplaceMissingFieldsWithNull` [#125764](https://github.com/elastic/elasticsearch/pull/125764) (issues: [#125764](https://github.com/elastic/elasticsearch/pull/125764), [#125764](https://github.com/elastic/elasticsearch/pull/125764), [#125764](https://github.com/elastic/elasticsearch/pull/125764)) +* Fix TDigestState.read CB leaks [#114303](https://github.com/elastic/elasticsearch/pull/114303) (issue: [#114194](https://github.com/elastic/elasticsearch/issues/114194)) +* Fix TopN row size estimate [#119476](https://github.com/elastic/elasticsearch/pull/119476) (issue: [#106956](https://github.com/elastic/elasticsearch/issues/106956)) +* Fix `AbstractShapeGeometryFieldMapperTests` [#119265](https://github.com/elastic/elasticsearch/pull/119265) (issue: [#119201](https://github.com/elastic/elasticsearch/issues/119201)) +* Fix `ReplaceMissingFieldsWithNull` [#125764](https://github.com/elastic/elasticsearch/pull/125764) (issues: [#126036](https://github.com/elastic/elasticsearch/issues/126036), [#121754](https://github.com/elastic/elasticsearch/issues/121754), [#126030](https://github.com/elastic/elasticsearch/issues/126030)) * Fix a bug in TOP [#121552](https://github.com/elastic/elasticsearch/pull/121552) -* Fix async stop sometimes not properly collecting result [#121843](https://github.com/elastic/elasticsearch/pull/121843) (issue: [#121843](https://github.com/elastic/elasticsearch/pull/121843)) +* Fix async stop sometimes not properly collecting result [#121843](https://github.com/elastic/elasticsearch/pull/121843) (issue: [#121249](https://github.com/elastic/elasticsearch/issues/121249)) * Fix attribute set equals [#118823](https://github.com/elastic/elasticsearch/pull/118823) -* Fix double lookup failure on ESQL [#115616](https://github.com/elastic/elasticsearch/pull/115616) (issue: [#115616](https://github.com/elastic/elasticsearch/pull/115616)) -* Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: [#123492](https://github.com/elastic/elasticsearch/pull/123492)) -* Fix queries with document level security on lookup indexes [#120617](https://github.com/elastic/elasticsearch/pull/120617) (issue: [#120617](https://github.com/elastic/elasticsearch/pull/120617)) -* Fix writing for LOOKUP status [#119296](https://github.com/elastic/elasticsearch/pull/119296) (issue: [#119296](https://github.com/elastic/elasticsearch/pull/119296)) -* Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: [#122601](https://github.com/elastic/elasticsearch/pull/122601)) +* Fix double lookup failure on ESQL [#115616](https://github.com/elastic/elasticsearch/pull/115616) (issue: [#111398](https://github.com/elastic/elasticsearch/issues/111398)) +* Fix function registry concurrency issues on constructor [#123492](https://github.com/elastic/elasticsearch/pull/123492) (issue: [#123430](https://github.com/elastic/elasticsearch/issues/123430)) +* Fix queries with document level security on lookup indexes [#120617](https://github.com/elastic/elasticsearch/pull/120617) (issue: [#120509](https://github.com/elastic/elasticsearch/issues/120509)) +* Fix writing for LOOKUP status [#119296](https://github.com/elastic/elasticsearch/pull/119296) (issue: [#119086](https://github.com/elastic/elasticsearch/issues/119086)) +* Implicit numeric casting for CASE/GREATEST/LEAST [#122601](https://github.com/elastic/elasticsearch/pull/122601) (issue: [#121890](https://github.com/elastic/elasticsearch/issues/121890)) * Lazy collection copying during node transform [#124424](https://github.com/elastic/elasticsearch/pull/124424) * Limit memory usage of `fold` [#118602](https://github.com/elastic/elasticsearch/pull/118602) * Limit size of query [#117898](https://github.com/elastic/elasticsearch/pull/117898) @@ -428,10 +428,10 @@ ES|QL: * Revert unwanted ES|QL lexer changes from PR #120354 [#120538](https://github.com/elastic/elasticsearch/pull/120538) * Revive inlinestats [#122257](https://github.com/elastic/elasticsearch/pull/122257) * Revive some more of inlinestats functionality [#123589](https://github.com/elastic/elasticsearch/pull/123589) -* Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: [#124001](https://github.com/elastic/elasticsearch/pull/124001)) +* Use a must boolean statement when pushing down to Lucene when scoring is also needed [#124001](https://github.com/elastic/elasticsearch/pull/124001) (issue: [#123967](https://github.com/elastic/elasticsearch/issues/123967)) Engine: -* Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: [#123010](https://github.com/elastic/elasticsearch/pull/123010)) +* Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` [#123010](https://github.com/elastic/elasticsearch/pull/123010) (issue: [#122974](https://github.com/elastic/elasticsearch/issues/122974)) Health: * Do not recommend increasing `max_shards_per_node` [#120458](https://github.com/elastic/elasticsearch/pull/120458) @@ -443,7 +443,7 @@ Indices APIs: * Add `?master_timeout` to `POST /_ilm/migrate_to_data_tiers` [#120883](https://github.com/elastic/elasticsearch/pull/120883) * Fix NPE in rolling over unknown target and return 404 [#125352](https://github.com/elastic/elasticsearch/pull/125352) * Fix broken yaml test `30_create_from` [#120662](https://github.com/elastic/elasticsearch/pull/120662) -* Include hidden indices in `DeprecationInfoAction` [#118035](https://github.com/elastic/elasticsearch/pull/118035) (issue: [#118035](https://github.com/elastic/elasticsearch/pull/118035)) +* Include hidden indices in `DeprecationInfoAction` [#118035](https://github.com/elastic/elasticsearch/pull/118035) (issue: [#118020](https://github.com/elastic/elasticsearch/issues/118020)) * Preventing `ConcurrentModificationException` when updating settings for more than one index [#126077](https://github.com/elastic/elasticsearch/pull/126077) * Updates the deprecation info API to not warn about system indices and data streams [#122951](https://github.com/elastic/elasticsearch/pull/122951) @@ -452,12 +452,12 @@ Inference: Infra/Core: * Epoch Millis Rounding Down and Not Up 2 [#118353](https://github.com/elastic/elasticsearch/pull/118353) -* Fix system data streams to be restorable from a snapshot [#124651](https://github.com/elastic/elasticsearch/pull/124651) (issue: [#124651](https://github.com/elastic/elasticsearch/pull/124651)) +* Fix system data streams to be restorable from a snapshot [#124651](https://github.com/elastic/elasticsearch/pull/124651) (issue: [#89261](https://github.com/elastic/elasticsearch/issues/89261)) * Have create index return a bad request on poor formatting [#123761](https://github.com/elastic/elasticsearch/pull/123761) * Include data streams when converting an existing resource to a system resource [#121392](https://github.com/elastic/elasticsearch/pull/121392) * System Index Migration Failure Results in a Non-Recoverable State [#122326](https://github.com/elastic/elasticsearch/pull/122326) -* System data streams are not being upgraded in the feature migration API [#124884](https://github.com/elastic/elasticsearch/pull/124884) (issue: [#124884](https://github.com/elastic/elasticsearch/pull/124884)) -* Wrap jackson exception on malformed json string [#114445](https://github.com/elastic/elasticsearch/pull/114445) (issue: [#114445](https://github.com/elastic/elasticsearch/pull/114445)) +* System data streams are not being upgraded in the feature migration API [#124884](https://github.com/elastic/elasticsearch/pull/124884) (issue: [#122949](https://github.com/elastic/elasticsearch/issues/122949)) +* Wrap jackson exception on malformed json string [#114445](https://github.com/elastic/elasticsearch/pull/114445) (issue: [#114142](https://github.com/elastic/elasticsearch/issues/114142)) Infra/Logging: * Move `SlowLogFieldProvider` instantiation to node construction [#117949](https://github.com/elastic/elasticsearch/pull/117949) @@ -470,16 +470,16 @@ Infra/Plugins: * Restrict agent entitlements to the system classloader unnamed module [#120546](https://github.com/elastic/elasticsearch/pull/120546) Infra/REST API: -* Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null. [#113413](https://github.com/elastic/elasticsearch/pull/113413) (issue: [#113413](https://github.com/elastic/elasticsearch/pull/113413)) +* Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null. [#113413](https://github.com/elastic/elasticsearch/pull/113413) (issue: [#113413](https://github.com/elastic/elasticsearch/issues/113413)) Infra/Scripting: * Register mustache size limit setting [#119291](https://github.com/elastic/elasticsearch/pull/119291) Infra/Settings: -* Don't allow secure settings in YML config (109115) [#115779](https://github.com/elastic/elasticsearch/pull/115779) (issue: [#115779](https://github.com/elastic/elasticsearch/pull/115779)) +* Don't allow secure settings in YML config (109115) [#115779](https://github.com/elastic/elasticsearch/pull/115779) (issue: [#109115](https://github.com/elastic/elasticsearch/issues/109115)) Ingest Node: -* Add warning headers for ingest pipelines containing special characters [#114837](https://github.com/elastic/elasticsearch/pull/114837) (issue: [#114837](https://github.com/elastic/elasticsearch/pull/114837)) +* Add warning headers for ingest pipelines containing special characters [#114837](https://github.com/elastic/elasticsearch/pull/114837) (issue: [#104411](https://github.com/elastic/elasticsearch/issues/104411)) * Fix geoip databases index access after system feature migration [#121196](https://github.com/elastic/elasticsearch/pull/121196) * Fix geoip databases index access after system feature migration (again) [#122938](https://github.com/elastic/elasticsearch/pull/122938) * Fix geoip databases index access after system feature migration (take 3) [#124604](https://github.com/elastic/elasticsearch/pull/124604) @@ -495,52 +495,52 @@ Machine Learning: * Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` [#125023](https://github.com/elastic/elasticsearch/pull/125023) * Fix get all inference endponts not returning multiple endpoints sharing model deployment [#121821](https://github.com/elastic/elasticsearch/pull/121821) * Fix serialising the inference update request [#122278](https://github.com/elastic/elasticsearch/pull/122278) -* Fixing bedrock event executor terminated cache issue [#118177](https://github.com/elastic/elasticsearch/pull/118177) (issue: [#118177](https://github.com/elastic/elasticsearch/pull/118177)) +* Fixing bedrock event executor terminated cache issue [#118177](https://github.com/elastic/elasticsearch/pull/118177) (issue: [#117916](https://github.com/elastic/elasticsearch/issues/117916)) * Fixing bug setting index when parsing Google Vertex AI results [#117287](https://github.com/elastic/elasticsearch/pull/117287) * Retry on streaming errors [#123076](https://github.com/elastic/elasticsearch/pull/123076) * Set Connect Timeout to 5s [#123272](https://github.com/elastic/elasticsearch/pull/123272) -* Set default similarity for Cohere model to cosine [#125370](https://github.com/elastic/elasticsearch/pull/125370) (issue: [#125370](https://github.com/elastic/elasticsearch/pull/125370)) +* Set default similarity for Cohere model to cosine [#125370](https://github.com/elastic/elasticsearch/pull/125370) (issue: [#122878](https://github.com/elastic/elasticsearch/issues/122878)) * Updating Inference Update API documentation to have the correct PUT method [#121048](https://github.com/elastic/elasticsearch/pull/121048) -* Wait for up to 2 seconds for yellow status before starting search [#115938](https://github.com/elastic/elasticsearch/pull/115938) (issues: [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938), [#115938](https://github.com/elastic/elasticsearch/pull/115938)) +* Wait for up to 2 seconds for yellow status before starting search [#115938](https://github.com/elastic/elasticsearch/pull/115938) (issues: [#107777](https://github.com/elastic/elasticsearch/issues/107777), [#105955](https://github.com/elastic/elasticsearch/issues/105955), [#107815](https://github.com/elastic/elasticsearch/issues/107815), [#112191](https://github.com/elastic/elasticsearch/issues/112191)) * [Inference API] Fix output stream ordering in `InferenceActionProxy` [#124225](https://github.com/elastic/elasticsearch/pull/124225) -* [Inference API] Fix unique ID message for inference ID matches trained model ID [#119543](https://github.com/elastic/elasticsearch/pull/119543) (issue: [#119543](https://github.com/elastic/elasticsearch/pull/119543)) +* [Inference API] Fix unique ID message for inference ID matches trained model ID [#119543](https://github.com/elastic/elasticsearch/pull/119543) (issue: [#111312](https://github.com/elastic/elasticsearch/issues/111312)) Mapping: * Avoid serializing empty `_source` fields in mappings [#122606](https://github.com/elastic/elasticsearch/pull/122606) * Enable New Semantic Text Format Only On Newly Created Indices [#121556](https://github.com/elastic/elasticsearch/pull/121556) * Fix Semantic Text 8.x Upgrade Bug [#125446](https://github.com/elastic/elasticsearch/pull/125446) -* Fix propagation of dynamic mapping parameter when applying `copy_to` [#121109](https://github.com/elastic/elasticsearch/pull/121109) (issue: [#121109](https://github.com/elastic/elasticsearch/pull/121109)) -* Fix realtime get of nested fields with synthetic source [#119575](https://github.com/elastic/elasticsearch/pull/119575) (issue: [#119575](https://github.com/elastic/elasticsearch/pull/119575)) -* Merge field mappers when updating mappings with [subobjects:false] [#120370](https://github.com/elastic/elasticsearch/pull/120370) (issue: [#120370](https://github.com/elastic/elasticsearch/pull/120370)) -* Merge template mappings properly during validation [#124784](https://github.com/elastic/elasticsearch/pull/124784) (issue: [#124784](https://github.com/elastic/elasticsearch/pull/124784)) -* Tweak `copy_to` handling in synthetic `_source` to account for nested objects [#120974](https://github.com/elastic/elasticsearch/pull/120974) (issue: [#120974](https://github.com/elastic/elasticsearch/pull/120974)) +* Fix propagation of dynamic mapping parameter when applying `copy_to` [#121109](https://github.com/elastic/elasticsearch/pull/121109) (issue: [#113049](https://github.com/elastic/elasticsearch/issues/113049)) +* Fix realtime get of nested fields with synthetic source [#119575](https://github.com/elastic/elasticsearch/pull/119575) (issue: [#119553](https://github.com/elastic/elasticsearch/issues/119553)) +* Merge field mappers when updating mappings with [subobjects:false] [#120370](https://github.com/elastic/elasticsearch/pull/120370) (issue: [#120216](https://github.com/elastic/elasticsearch/issues/120216)) +* Merge template mappings properly during validation [#124784](https://github.com/elastic/elasticsearch/pull/124784) (issue: [#123372](https://github.com/elastic/elasticsearch/issues/123372)) +* Tweak `copy_to` handling in synthetic `_source` to account for nested objects [#120974](https://github.com/elastic/elasticsearch/pull/120974) (issue: [#120831](https://github.com/elastic/elasticsearch/issues/120831)) Network: -* Remove ChunkedToXContentBuilder [#119310](https://github.com/elastic/elasticsearch/pull/119310) (issue: [#119310](https://github.com/elastic/elasticsearch/pull/119310)) +* Remove ChunkedToXContentBuilder [#119310](https://github.com/elastic/elasticsearch/pull/119310) (issue: [#118647](https://github.com/elastic/elasticsearch/issues/118647)) Ranking: * Fix LTR query feature with phrases (and two-phase) queries [#125103](https://github.com/elastic/elasticsearch/pull/125103) Search: * Catch and handle disconnect exceptions in search [#115836](https://github.com/elastic/elasticsearch/pull/115836) -* Fix leak in `DfsQueryPhase` and introduce search disconnect stress test [#116060](https://github.com/elastic/elasticsearch/pull/116060) (issue: [#116060](https://github.com/elastic/elasticsearch/pull/116060)) +* Fix leak in `DfsQueryPhase` and introduce search disconnect stress test [#116060](https://github.com/elastic/elasticsearch/pull/116060) (issue: [#115056](https://github.com/elastic/elasticsearch/issues/115056)) * Fix/QueryBuilderBWCIT_muted_test [#117831](https://github.com/elastic/elasticsearch/pull/117831) -* Handle long overflow in dates [#124048](https://github.com/elastic/elasticsearch/pull/124048) (issue: [#124048](https://github.com/elastic/elasticsearch/pull/124048)) -* Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: [#122357](https://github.com/elastic/elasticsearch/pull/122357)) +* Handle long overflow in dates [#124048](https://github.com/elastic/elasticsearch/pull/124048) (issue: [#112483](https://github.com/elastic/elasticsearch/issues/112483)) +* Handle search timeout in `SuggestPhase` [#122357](https://github.com/elastic/elasticsearch/pull/122357) (issue: [#122186](https://github.com/elastic/elasticsearch/issues/122186)) * In this pr, a 400 error is returned when _source / _seq_no / _feature / _nested_path / _field_names is requested, rather a 5xx [#117229](https://github.com/elastic/elasticsearch/pull/117229) * Inconsistency in the _analyzer api when the index is not included [#115930](https://github.com/elastic/elasticsearch/pull/115930) -* Let MLTQuery throw IAE when no analyzer is set [#124662](https://github.com/elastic/elasticsearch/pull/124662) (issue: [#124662](https://github.com/elastic/elasticsearch/pull/124662)) -* Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` [#125650](https://github.com/elastic/elasticsearch/pull/125650) (issue: [#125650](https://github.com/elastic/elasticsearch/pull/125650)) +* Let MLTQuery throw IAE when no analyzer is set [#124662](https://github.com/elastic/elasticsearch/pull/124662) (issue: [#124562](https://github.com/elastic/elasticsearch/issues/124562)) +* Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` [#125650](https://github.com/elastic/elasticsearch/pull/125650) (issue: [#125483](https://github.com/elastic/elasticsearch/issues/125483)) * Log stack traces on data nodes before they are cleared for transport [#125732](https://github.com/elastic/elasticsearch/pull/125732) * Minor-Fixes Support 7x segments as archive in 8x / 9x [#125666](https://github.com/elastic/elasticsearch/pull/125666) * Re-enable parallel collection for field sorted top hits [#125916](https://github.com/elastic/elasticsearch/pull/125916) * Remove duplicate code in ESIntegTestCase [#120799](https://github.com/elastic/elasticsearch/pull/120799) -* SearchStatesIt failures reported by CI [#117618](https://github.com/elastic/elasticsearch/pull/117618) (issues: [#117618](https://github.com/elastic/elasticsearch/pull/117618), [#117618](https://github.com/elastic/elasticsearch/pull/117618)) +* SearchStatesIt failures reported by CI [#117618](https://github.com/elastic/elasticsearch/pull/117618) (issues: [#116617](https://github.com/elastic/elasticsearch/issues/116617), [#116618](https://github.com/elastic/elasticsearch/issues/116618)) * Skip fetching _inference_fields field in legacy semantic_text format [#121720](https://github.com/elastic/elasticsearch/pull/121720) -* Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. [#119503](https://github.com/elastic/elasticsearch/pull/119503) (issue: [#119503](https://github.com/elastic/elasticsearch/pull/119503)) -* Test/107515 restore template with match only text mapper it fail [#120392](https://github.com/elastic/elasticsearch/pull/120392) (issue: [#120392](https://github.com/elastic/elasticsearch/pull/120392)) -* Updated Date Range to Follow Documentation When Assuming Missing Values [#112258](https://github.com/elastic/elasticsearch/pull/112258) (issue: [#112258](https://github.com/elastic/elasticsearch/pull/112258)) -* `CrossClusterIT` `testCancel` failure [#117750](https://github.com/elastic/elasticsearch/pull/117750) (issue: [#117750](https://github.com/elastic/elasticsearch/pull/117750)) +* Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. [#119503](https://github.com/elastic/elasticsearch/pull/119503) (issue: [#117042](https://github.com/elastic/elasticsearch/issues/117042)) +* Test/107515 restore template with match only text mapper it fail [#120392](https://github.com/elastic/elasticsearch/pull/120392) (issue: [#107515](https://github.com/elastic/elasticsearch/issues/107515)) +* Updated Date Range to Follow Documentation When Assuming Missing Values [#112258](https://github.com/elastic/elasticsearch/pull/112258) (issue: [#111484](https://github.com/elastic/elasticsearch/issues/111484)) +* `CrossClusterIT` `testCancel` failure [#117750](https://github.com/elastic/elasticsearch/pull/117750) (issue: [#108061](https://github.com/elastic/elasticsearch/issues/108061)) * `SearchServiceTests.testParseSourceValidation` failure [#117963](https://github.com/elastic/elasticsearch/pull/117963) Snapshot/Restore: @@ -548,23 +548,23 @@ Snapshot/Restore: * Fork post-snapshot-delete cleanup off master thread [#122731](https://github.com/elastic/elasticsearch/pull/122731) * Retry throttled snapshot deletions [#113237](https://github.com/elastic/elasticsearch/pull/113237) * This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. [#124931](https://github.com/elastic/elasticsearch/pull/124931) -* Use the system index descriptor in the snapshot blob cache cleanup task [#120937](https://github.com/elastic/elasticsearch/pull/120937) (issue: [#120937](https://github.com/elastic/elasticsearch/pull/120937)) +* Use the system index descriptor in the snapshot blob cache cleanup task [#120937](https://github.com/elastic/elasticsearch/pull/120937) (issue: [#120518](https://github.com/elastic/elasticsearch/issues/120518)) Store: * Do not capture `ClusterChangedEvent` in `IndicesStore` call to #onClusterStateShardsClosed [#120193](https://github.com/elastic/elasticsearch/pull/120193) Suggesters: -* Return an empty suggestion when suggest phase times out [#122575](https://github.com/elastic/elasticsearch/pull/122575) (issue: [#122575](https://github.com/elastic/elasticsearch/pull/122575)) +* Return an empty suggestion when suggest phase times out [#122575](https://github.com/elastic/elasticsearch/pull/122575) (issue: [#122548](https://github.com/elastic/elasticsearch/issues/122548)) Transform: -* If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias [#122074](https://github.com/elastic/elasticsearch/pull/122074) (issue: [#122074](https://github.com/elastic/elasticsearch/pull/122074)) +* If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias [#122074](https://github.com/elastic/elasticsearch/pull/122074) (issue: [#121913](https://github.com/elastic/elasticsearch/issues/121913)) Vector Search: * Apply default k for knn query eagerly [#118774](https://github.com/elastic/elasticsearch/pull/118774) -* Fix `bbq_hnsw` merge file cleanup on random IO exceptions [#119691](https://github.com/elastic/elasticsearch/pull/119691) (issue: [#119691](https://github.com/elastic/elasticsearch/pull/119691)) -* Knn vector rescoring to sort score docs [#122653](https://github.com/elastic/elasticsearch/pull/122653) (issue: [#122653](https://github.com/elastic/elasticsearch/pull/122653)) +* Fix `bbq_hnsw` merge file cleanup on random IO exceptions [#119691](https://github.com/elastic/elasticsearch/pull/119691) (issue: [#119392](https://github.com/elastic/elasticsearch/issues/119392)) +* Knn vector rescoring to sort score docs [#122653](https://github.com/elastic/elasticsearch/pull/122653) (issue: [#119711](https://github.com/elastic/elasticsearch/issues/119711)) * Return appropriate error on null dims update instead of npe [#125716](https://github.com/elastic/elasticsearch/pull/125716) Watcher: -* Watcher history index has too many indexed fields - [#117701](https://github.com/elastic/elasticsearch/pull/117701) (issue: [#117701](https://github.com/elastic/elasticsearch/pull/117701)) +* Watcher history index has too many indexed fields - [#117701](https://github.com/elastic/elasticsearch/pull/117701) (issue: [#71479](https://github.com/elastic/elasticsearch/issues/71479)) From 51215d0f111e9e3f455db4e157ac812ae3e3192c Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 14 Apr 2025 14:32:30 -0400 Subject: [PATCH 18/34] Fix more issue links --- .../main/resources/templates/breaking-changes.md | 2 +- .../src/main/resources/templates/deprecations.md | 2 +- docs/release-notes/breaking-changes.md | 16 ++++++++-------- docs/release-notes/deprecations.md | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.md b/build-tools-internal/src/main/resources/templates/breaking-changes.md index 9f0dc982c0afa..918c31775be41 100644 --- a/build-tools-internal/src/main/resources/templates/breaking-changes.md +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.md @@ -32,7 +32,7 @@ for(bundle in changelogBundles) { print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" if (change.issues != null && change.issues.empty == false) { print change.issues.size() == 1 ? " (issue: " : " (issues: " - print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") + print change.issues.collect { "[#${it}](https://github.com/elastic/elasticsearch/issues/${it})" }.join(", ") print ")" } print "\n" diff --git a/build-tools-internal/src/main/resources/templates/deprecations.md b/build-tools-internal/src/main/resources/templates/deprecations.md index 4470d1f5a78f0..795ff81d9f56a 100644 --- a/build-tools-internal/src/main/resources/templates/deprecations.md +++ b/build-tools-internal/src/main/resources/templates/deprecations.md @@ -34,7 +34,7 @@ for(bundle in changelogBundles) { print "* ${change.summary} [#${change.pr}](https://github.com/elastic/elasticsearch/pull/${change.pr})" if (change.issues != null && change.issues.empty == false) { print change.issues.size() == 1 ? " (issue: " : " (issues: " - print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") + print change.issues.collect { "[#${it}](https://github.com/elastic/elasticsearch/issues/${it})" }.join(", ") print ")" } print "\n" diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index a80e6fa606661..85a4274e62860 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -22,12 +22,12 @@ Aggregations: Allocation: * Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831) * Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting [#114207](https://github.com/elastic/elasticsearch/pull/114207) -* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: {es-issue}88978[#88978]) +* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: [#88978](https://github.com/elastic/elasticsearch/issues/88978)) Analysis: * Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146) * The 'german2' stemmer is now an alias for the 'german' snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614) -* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: {es-issue}113050[#113050]) +* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: [#113050](https://github.com/elastic/elasticsearch/issues/113050)) * The Korean dictionary for Nori has been updated [#114124](https://github.com/elastic/elasticsearch/pull/114124) Authentication: @@ -55,32 +55,32 @@ Indices APIs: Infra/Core: * Change Elasticsearch timeouts to 429 response instead of 5xx [#116026](https://github.com/elastic/elasticsearch/pull/116026) * Limit `ByteSizeUnit` to 2 decimals [#120142](https://github.com/elastic/elasticsearch/pull/120142) -* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: {es-issue}104574[#104574]) +* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: [#104574](https://github.com/elastic/elasticsearch/issues/104574)) * Remove any references to org.elasticsearch.core.RestApiVersion#V_7 [#118103](https://github.com/elastic/elasticsearch/pull/118103) Infra/Logging: -* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: {es-issue}83251[#83251]) -* Rename deprecation index template [#125606](https://github.com/elastic/elasticsearch/pull/125606) (issue: {es-issue}125445[#125445]) +* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: [#83251](https://github.com/elastic/elasticsearch/issues/83251)) +* Rename deprecation index template [#125606](https://github.com/elastic/elasticsearch/pull/125606) (issue: [#125445](https://github.com/elastic/elasticsearch/issues/125445)) Infra/Metrics: * Deprecated tracing.apm.* settings got removed. [#119926](https://github.com/elastic/elasticsearch/pull/119926) Infra/REST API: -* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: {es-issue}89387[#89387]) +* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: [#89387](https://github.com/elastic/elasticsearch/issues/89387)) Ingest Node: * Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077) * Remove ignored fallback option on GeoIP processor [#116112](https://github.com/elastic/elasticsearch/pull/116112) Logs: -* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: {es-issue}106489[#106489]) +* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: [#106489](https://github.com/elastic/elasticsearch/issues/106489)) Machine Learning: * Disable machine learning on macOS x86_64 [#104125](https://github.com/elastic/elasticsearch/pull/104125) Mapping: * Remove support for type, fields, `copy_to` and boost in metadata field definition [#118825](https://github.com/elastic/elasticsearch/pull/118825) -* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: {es-issue}118596[#118596]) +* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: [#118596](https://github.com/elastic/elasticsearch/issues/118596)) Search: * Adjust `random_score` default field to `_seq_no` field [#118671](https://github.com/elastic/elasticsearch/pull/118671) diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index 1d353648292ec..b835466a91452 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -19,13 +19,13 @@ To give you insight into what deprecated features you’re using, {{es}}: ## 9.0.0 [elasticsearch-900-deprecations] ES|QL: -* Drop support for brackets from METADATA syntax [#119846](https://github.com/elastic/elasticsearch/pull/119846) (issue: {es-issue}115401[#115401]) +* Drop support for brackets from METADATA syntax [#119846](https://github.com/elastic/elasticsearch/pull/119846) (issue: [#115401](https://github.com/elastic/elasticsearch/issues/115401)) Ingest Node: * Fix `_type` deprecation on simulate pipeline API [#116259](https://github.com/elastic/elasticsearch/pull/116259) Machine Learning: -* Add deprecation warning for flush API [#121667](https://github.com/elastic/elasticsearch/pull/121667) (issue: {es-issue}121506[#121506]) +* Add deprecation warning for flush API [#121667](https://github.com/elastic/elasticsearch/pull/121667) (issue: [#121506](https://github.com/elastic/elasticsearch/issues/121506)) * Removing index alias creation for deprecated transforms notification index [#117583](https://github.com/elastic/elasticsearch/pull/117583) * [Inference API] Deprecate elser service [#113216](https://github.com/elastic/elasticsearch/pull/113216) From 622bed4e914389f4c02eccaeec48cfe346bcc8d4 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 28 Apr 2025 17:43:36 -0400 Subject: [PATCH 19/34] Add prelim 9.0.1 release notes --- docs/release-notes/breaking-changes.md | 4 + .../release-notes/changelog-bundles/9.0.1.yml | 132 ++++++++++++++++++ docs/release-notes/deprecations.md | 4 + docs/release-notes/index.md | 65 +++++++++ 4 files changed, 205 insertions(+) create mode 100644 docs/release-notes/changelog-bundles/9.0.1.yml diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index 85a4274e62860..543855a350331 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -14,6 +14,10 @@ To learn how to upgrade, check out . % ## Next version [elasticsearch-nextversion-breaking-changes] +## 9.0.1 [elasticsearch-901-breaking-changes] + +No breaking changes in this version. + ## 9.0.0 [elasticsearch-900-breaking-changes] Aggregations: diff --git a/docs/release-notes/changelog-bundles/9.0.1.yml b/docs/release-notes/changelog-bundles/9.0.1.yml new file mode 100644 index 0000000000000..806a1a5a97d4e --- /dev/null +++ b/docs/release-notes/changelog-bundles/9.0.1.yml @@ -0,0 +1,132 @@ +version: 9.0.1 +generated: 2025-04-28T21:41:06.757953Z +changelogs: + - pr: 125694 + summary: LTR score bounding + area: Ranking + type: bug + issues: [] + - pr: 126273 + summary: Fix LTR rescorer with model alias + area: Ranking + type: bug + issues: [] + - pr: 126310 + summary: Add Issuer to failed SAML Signature validation logs when available + area: Security + type: enhancement + issues: + - 111022 + - pr: 126342 + summary: Enable sort optimization on float and `half_float` + area: Search + type: enhancement + issues: [] + - pr: 126583 + summary: Cancel expired async search task when a remote returns its results + area: CCS + type: bug + issues: [] + - pr: 126605 + summary: Fix equality bug in `WaitForIndexColorStep` + area: ILM+SLM + type: bug + issues: [] + - pr: 126614 + summary: Fix join masking eval + area: ES|QL + type: bug + issues: [] + - pr: 126637 + summary: Improve resiliency of `UpdateTimeSeriesRangeService` + area: TSDB + type: bug + issues: [] + - pr: 126686 + summary: Fix race condition in `RestCancellableNodeClient` + area: Task Management + type: bug + issues: + - 88201 + - pr: 126729 + summary: Use terminal reader in keystore add command + area: Infra/CLI + type: bug + issues: + - 98115 + - pr: 126778 + summary: Fix bbq quantization algorithm but for differently distributed components + area: Vector Search + type: bug + issues: [] + - pr: 126783 + summary: Fix shard size of initializing restored shard + area: Allocation + type: bug + issues: + - 105331 + - pr: 126806 + summary: Workaround max name limit imposed by Jackson 2.17 + area: Infra/Core + type: bug + issues: [] + - pr: 126850 + summary: "[otel-data] Bump plugin version to release _metric_names_hash changes" + area: Data streams + type: bug + issues: [] + - pr: 126852 + summary: "Validation checks on paths allowed for 'files' entitlements. Restrict the paths we allow access to, forbidding plugins to specify/request entitlements for reading or writing to specific protected directories." + area: Infra/Core + type: enhancement + issues: [] + - pr: 126858 + summary: Leverage threadpool schedule for inference api to avoid long running thread + area: Machine Learning + type: bug + issues: + - 126853 + - pr: 126884 + summary: Rare terms aggregation false **positive** fix + area: Aggregations + type: bug + issues: [] + - pr: 126889 + summary: Rework uniquify to not use iterators + area: Infra/Core + type: bug + issues: + - 126883 + - pr: 126911 + summary: Fix `vec_caps` to test for OS support too (on x64) + area: Vector Search + type: bug + issues: + - 126809 + - pr: 126930 + summary: Adding missing `onFailure` call for Inference API start model request + area: Machine Learning + type: bug + issues: [] + - pr: 126990 + summary: "Fix: consider case sensitiveness differences in Windows/Unix-like filesystems for files entitlements" + area: Infra/Core + type: bug + issues: + - 127047 + - pr: 127146 + summary: Fix sneaky bug in single value query + area: ES|QL + type: bug + issues: [] + - pr: 127225 + summary: Fix count optimization with pushable union types + area: ES|QL + type: bug + issues: + - 127200 + - pr: 127353 + summary: Updating tika to 2.9.3 + area: Ingest Node + type: upgrade + issues: [] diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index b835466a91452..14a3215381c83 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -16,6 +16,10 @@ To give you insight into what deprecated features you’re using, {{es}}: % ## Next version [elasticsearch-nextversion-deprecations] +## 9.0.1 [elasticsearch-901-deprecations] + +No deprecations in this version. + ## 9.0.0 [elasticsearch-900-deprecations] ES|QL: diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index e659feea03c9e..84f13d74b0802 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -21,6 +21,71 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * +## 9.0.1 [elasticsearch-901-release-notes] + +### Features and enhancements [elasticsearch-901-features-enhancements] + +Infra/Core: +* Validation checks on paths allowed for 'files' entitlements. Restrict the paths we allow access to, forbidding plugins to specify/request entitlements for reading or writing to specific protected directories. [#126852](https://github.com/elastic/elasticsearch/pull/126852) + +Ingest Node: +* Updating tika to 2.9.3 [#127353](https://github.com/elastic/elasticsearch/pull/127353) + +Search: +* Enable sort optimization on float and `half_float` [#126342](https://github.com/elastic/elasticsearch/pull/126342) + +Security: +* Add Issuer to failed SAML Signature validation logs when available [#126310](https://github.com/elastic/elasticsearch/pull/126310) (issue: [#111022](https://github.com/elastic/elasticsearch/issues/111022)) + +### Fixes [elasticsearch-901-fixes] + +Aggregations: +* Rare terms aggregation false **positive** fix [#126884](https://github.com/elastic/elasticsearch/pull/126884) + +Allocation: +* Fix shard size of initializing restored shard [#126783](https://github.com/elastic/elasticsearch/pull/126783) (issue: [#105331](https://github.com/elastic/elasticsearch/issues/105331)) + +CCS: +* Cancel expired async search task when a remote returns its results [#126583](https://github.com/elastic/elasticsearch/pull/126583) + +Data streams: +* [otel-data] Bump plugin version to release _metric_names_hash changes [#126850](https://github.com/elastic/elasticsearch/pull/126850) + +ES|QL: +* Fix count optimization with pushable union types [#127225](https://github.com/elastic/elasticsearch/pull/127225) (issue: [#127200](https://github.com/elastic/elasticsearch/issues/127200)) +* Fix join masking eval [#126614](https://github.com/elastic/elasticsearch/pull/126614) +* Fix sneaky bug in single value query [#127146](https://github.com/elastic/elasticsearch/pull/127146) + +ILM+SLM: +* Fix equality bug in `WaitForIndexColorStep` [#126605](https://github.com/elastic/elasticsearch/pull/126605) + +Infra/CLI: +* Use terminal reader in keystore add command [#126729](https://github.com/elastic/elasticsearch/pull/126729) (issue: [#98115](https://github.com/elastic/elasticsearch/issues/98115)) + +Infra/Core: +* Fix: consider case sensitiveness differences in Windows/Unix-like filesystems for files entitlements [#126990](https://github.com/elastic/elasticsearch/pull/126990) (issue: [#127047](https://github.com/elastic/elasticsearch/issues/127047)) +* Rework uniquify to not use iterators [#126889](https://github.com/elastic/elasticsearch/pull/126889) (issue: [#126883](https://github.com/elastic/elasticsearch/issues/126883)) +* Workaround max name limit imposed by Jackson 2.17 [#126806](https://github.com/elastic/elasticsearch/pull/126806) + +Machine Learning: +* Adding missing `onFailure` call for Inference API start model request [#126930](https://github.com/elastic/elasticsearch/pull/126930) +* Leverage threadpool schedule for inference api to avoid long running thread [#126858](https://github.com/elastic/elasticsearch/pull/126858) (issue: [#126853](https://github.com/elastic/elasticsearch/issues/126853)) + +Ranking: +* Fix LTR rescorer with model alias [#126273](https://github.com/elastic/elasticsearch/pull/126273) +* LTR score bounding [#125694](https://github.com/elastic/elasticsearch/pull/125694) + +TSDB: +* Improve resiliency of `UpdateTimeSeriesRangeService` [#126637](https://github.com/elastic/elasticsearch/pull/126637) + +Task Management: +* Fix race condition in `RestCancellableNodeClient` [#126686](https://github.com/elastic/elasticsearch/pull/126686) (issue: [#88201](https://github.com/elastic/elasticsearch/issues/88201)) + +Vector Search: +* Fix `vec_caps` to test for OS support too (on x64) [#126911](https://github.com/elastic/elasticsearch/pull/126911) (issue: [#126809](https://github.com/elastic/elasticsearch/issues/126809)) +* Fix bbq quantization algorithm but for differently distributed components [#126778](https://github.com/elastic/elasticsearch/pull/126778) + + ## 9.0.0 [elasticsearch-900-release-notes] ### Highlights [elasticsearch-900-highlights] From 801ad688b99a5203aea1553e4ea971bc0d99fda0 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 29 Apr 2025 16:44:45 -0400 Subject: [PATCH 20/34] Update 9.0.0 docs with changes from main --- .../src/main/resources/templates/index.md | 2 - .../release-notes/changelog-bundles/9.0.0.yml | 39 ++++++++++--------- docs/release-notes/index.md | 28 ++++++------- 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md index bcebcf83c1fc8..e2c9e463d87ff 100644 --- a/build-tools-internal/src/main/resources/templates/index.md +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -40,8 +40,6 @@ for (highlights in [notableHighlights, nonNotableHighlights]) { for (highlight in highlights) { %> ::::{dropdown} ${highlight.title} ${highlight.body.trim()} - -For more information, check [PR #${highlight.pr}](https://github.com/elastic/elasticsearch/pull/${highlight.pr}). :::: <% } } diff --git a/docs/release-notes/changelog-bundles/9.0.0.yml b/docs/release-notes/changelog-bundles/9.0.0.yml index 1f1d1aac2847c..1766991b57b30 100644 --- a/docs/release-notes/changelog-bundles/9.0.0.yml +++ b/docs/release-notes/changelog-bundles/9.0.0.yml @@ -165,7 +165,7 @@ changelogs: breaking: area: Analysis title: The "german2" snowball stemmer is now an alias for the "german" stemmer - details: "Lucene 10 has merged the improved \"german2\" snowball language stemmer with the \"german\" stemmer. For Elasticsearch, \"german2\" is now a deprecated alias for \"german\". This may results in slightly different tokens being generated for terms with umlaut substitution (like \"ue\" for \"ü\" etc...)" + details: 'Lucene 10 has merged the improved "german2" snowball language stemmer with the "german" stemmer. For Elasticsearch, "german2" is now a deprecated alias for "german". This may results in slightly different tokens being generated for terms with umlaut substitution (like "ue" for "ü" etc...)' impact: Replace usages of "german2" with "german" in analysis configuration. Old indices that use the "german" stemmer should be reindexed if possible. notable: false essSettingChange: false @@ -640,7 +640,7 @@ changelogs: breaking: area: Transform title: Remove `data_frame_transforms` roles - details: '`data_frame_transforms_admin` and `data_frame_transforms_user` were deprecated in Elasticsearch 7 and are being removed in Elasticsearch 9. `data_frame_transforms_admin` is now `transform_admin`. `data_frame_transforms_user` is now `transform_user`. Users must call the `_update` API to replace the permissions on the Transform before the Transform can be started.' + details: "`data_frame_transforms_admin` and `data_frame_transforms_user` were deprecated in Elasticsearch 7 and are being removed in Elasticsearch 9. `data_frame_transforms_admin` is now `transform_admin`. `data_frame_transforms_user` is now `transform_user`. Users must call the `_update` API to replace the permissions on the Transform before the Transform can be started." impact: "Transforms created with either the `data_frame_transforms_admin` or the `data_frame_transforms_user` role will fail to start. The Transform will remain in a `stopped` state, and its health will be red while displaying permission failures." notable: false essSettingChange: false @@ -723,7 +723,7 @@ changelogs: issues: - 105135 - pr: 117750 - summary: '`CrossClusterIT` `testCancel` failure' + summary: "`CrossClusterIT` `testCancel` failure" area: Search type: bug issues: @@ -788,7 +788,7 @@ changelogs: type: bug issues: [] - pr: 117963 - summary: '`SearchServiceTests.testParseSourceValidation` failure' + summary: "`SearchServiceTests.testParseSourceValidation` failure" area: Search type: bug issues: [] @@ -969,7 +969,7 @@ changelogs: type: enhancement issues: [] - pr: 118435 - summary: '`_score` should not be a reserved attribute in ES|QL' + summary: "`_score` should not be a reserved attribute in ES|QL" area: ES|QL type: enhancement issues: @@ -1056,7 +1056,7 @@ changelogs: type: enhancement issues: [] - pr: 118681 - summary: '`ConnectTransportException` returns retryable BAD_GATEWAY' + summary: "`ConnectTransportException` returns retryable BAD_GATEWAY" area: Network type: enhancement issues: @@ -1089,8 +1089,11 @@ changelogs: issues: [] highlight: notable: true - title: Add new experimental `rank_vectors` mapping for late-interaction second order ranking - body: "Late-interaction models are powerful rerankers. While their size and overall cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali." + title: rank_vectors field type is now available for late-interaction ranking + body: | + [`rank_vectors`](../reference/elasticsearch/mapping-reference/rank-vectors.md) is a new field type released as an experimental feature in Elasticsearch 9.0. It is designed to be used with dense vectors and allows for late-interaction second order ranking. + + Late-interaction models are powerful rerankers. While their size and overall cost doesn’t lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali. pr: 118804 - pr: 118823 summary: Fix attribute set equals @@ -1922,16 +1925,14 @@ changelogs: - 121185 highlight: notable: true - title: Enable LOOKUP JOIN in non-snapshot builds + title: "ES|QL LOOKUP JOIN is now available in technical preview" body: |- - This effectively releases LOOKUP JOIN into tech preview. Docs will - follow in a separate PR. - - - Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds. - - Remove the grammar for the unsupported `| JOIN ...` command (without `LOOKUP` as first keyword). The way the lexer modes work, otherwise we'd also have to enable `| JOIN ...` syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages. - - Remove grammar for `LOOKUP JOIN index AS ...` because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries. + [LOOKUP JOIN](../reference/query-languages/esql/esql-commands.md) is now available in technical preview. LOOKUP JOIN combines data from your ES|QL queries with matching records from a lookup index, enabling you to: - Also fix https://github.com/elastic/elasticsearch/issues/121185 + - Enrich your search results with reference data + - Speed up root-cause analysis and security investigations + - Join data across indices without complex queries + - Reduce operational overhead when correlating events pr: 121193 - pr: 121196 summary: Fix geoip databases index access after system feature migration @@ -1944,7 +1945,7 @@ changelogs: type: bug issues: [] - pr: 121325 - summary: '`ReindexDataStreamIndex` bug in assertion caused by reference equality' + summary: "`ReindexDataStreamIndex` bug in assertion caused by reference equality" area: Data streams type: bug issues: [] @@ -2342,8 +2343,8 @@ changelogs: issues: [] highlight: notable: true - title: Release semantic_text as a GA feature - body: semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. + title: The semantic_text field type is now GA + body: "[`semantic_text`](../reference/elasticsearch/mapping-reference/semantic-text.md) is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time." pr: 124669 - pr: 124739 summary: Improve rolling up metrics diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 84f13d74b0802..8fb66ab3754f0 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -90,29 +90,23 @@ Vector Search: ### Highlights [elasticsearch-900-highlights] -::::{dropdown} Add new experimental `rank_vectors` mapping for late-interaction second order ranking -Late-interaction models are powerful rerankers. While their size and overall cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali. +::::{dropdown} rank_vectors field type is now available for late-interaction ranking +[`rank_vectors`](../reference/elasticsearch/mapping-reference/rank-vectors.md) is a new field type released as an experimental feature in Elasticsearch 9.0. It is designed to be used with dense vectors and allows for late-interaction second order ranking. -For more information, check [PR #118804](https://github.com/elastic/elasticsearch/pull/118804). +Late-interaction models are powerful rerankers. While their size and overall cost doesn’t lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali. :::: -::::{dropdown} Enable LOOKUP JOIN in non-snapshot builds -This effectively releases LOOKUP JOIN into tech preview. Docs will -follow in a separate PR. +::::{dropdown} ES|QL LOOKUP JOIN is now available in technical preview +[LOOKUP JOIN](../reference/query-languages/esql/esql-commands.md) is now available in technical preview. LOOKUP JOIN combines data from your ES|QL queries with matching records from a lookup index, enabling you to: -- Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds. -- Remove the grammar for the unsupported `| JOIN ...` command (without `LOOKUP` as first keyword). The way the lexer modes work, otherwise we'd also have to enable `| JOIN ...` syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages. -- Remove grammar for `LOOKUP JOIN index AS ...` because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries. - -Also fix https://github.com/elastic/elasticsearch/issues/121185 - -For more information, check [PR #121193](https://github.com/elastic/elasticsearch/pull/121193). +- Enrich your search results with reference data +- Speed up root-cause analysis and security investigations +- Join data across indices without complex queries +- Reduce operational overhead when correlating events :::: -::::{dropdown} Release semantic_text as a GA feature -semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. - -For more information, check [PR #124669](https://github.com/elastic/elasticsearch/pull/124669). +::::{dropdown} The semantic_text field type is now GA +[`semantic_text`](../reference/elasticsearch/mapping-reference/semantic-text.md) is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. :::: ### Features and enhancements [elasticsearch-900-features-enhancements] From b642cc55f692cbec62993b6d395770ecfb983160 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 29 Apr 2025 16:50:28 -0400 Subject: [PATCH 21/34] Fix template --- .../src/main/resources/templates/breaking-changes.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.md b/build-tools-internal/src/main/resources/templates/breaking-changes.md index 918c31775be41..b511e93471579 100644 --- a/build-tools-internal/src/main/resources/templates/breaking-changes.md +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.md @@ -10,8 +10,6 @@ Breaking changes can impact your Elastic applications, potentially disrupting no If you are migrating from a version prior to version 9.0, you must first upgrade to the last 8.x version available. To learn how to upgrade, check out [Upgrade](docs-content://deploy-manage/upgrade.md). -To learn how to upgrade, check out . - % ## Next version [elasticsearch-nextversion-breaking-changes] <% for(bundle in changelogBundles) { From 80d16ecfb322a0419f39acd3d76770f3539e97eb Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 2 May 2025 15:35:57 -0400 Subject: [PATCH 22/34] WIP handling multiple releases in the docs, and update release notes for 9.0.1 --- .../release/BundleChangelogsTask.java | 65 +++++++++++++++++-- .../internal/release/ChangelogBundle.java | 6 +- .../release/GenerateReleaseNotesTask.java | 27 ++------ .../internal/release/PruneChangelogsTask.java | 2 +- .../internal/release/ReleaseToolsPlugin.java | 3 + .../src/main/resources/templates/index.md | 16 +++-- docs/release-notes/breaking-changes.md | 2 - .../release-notes/changelog-bundles/9.0.0.yml | 1 + .../release-notes/changelog-bundles/9.0.1.yml | 18 ++++- docs/release-notes/index.md | 14 +++- 10 files changed, 114 insertions(+), 40 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index a9aa33e69078a..a6013b720cb5a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import org.elasticsearch.gradle.Version; import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.Directory; @@ -28,6 +29,7 @@ import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.options.Option; import org.gradle.process.ExecOperations; import java.io.File; @@ -38,6 +40,7 @@ import java.util.List; import java.util.Properties; +import javax.annotation.Nullable; import javax.inject.Inject; import static java.util.stream.Collectors.toList; @@ -52,6 +55,29 @@ public class BundleChangelogsTask extends DefaultTask { private final GitWrapper gitWrapper; + @Nullable + private String branch; + private boolean updateExisting; + private boolean finalize; + + @Option( + option = "update-existing", + description = "Only update entries that are already in the bundle. Useful for updating the bundle after a BC has been cut." + ) + public void setUpdateExisting(boolean updateExisting) { + this.updateExisting = updateExisting; + } + + @Option(option = "branch", description = "Branch (or other ref) to use for generating the changelog bundle.") + public void setBranch(String branch) { + this.branch = branch; + } + + @Option(option = "finalize", description = "Specify that the bundle is finalized, i.e. that the version has been released.") + public void setFinalize(boolean finalize) { + this.finalize = finalize; + } + private static final ObjectMapper yamlMapper = new ObjectMapper( new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) .disable(YAMLGenerator.Feature.SPLIT_LINES) @@ -72,13 +98,16 @@ public BundleChangelogsTask(ObjectFactory objectFactory, ExecOperations execOper @TaskAction public void executeTask() throws IOException { + if (branch == null) { + throw new IllegalArgumentException("'branch' not specified."); + } + final String upstreamRemote = gitWrapper.getUpstream(); - String ref = "9.0-fix-release-notes"; try { - checkoutChangelogs(gitWrapper, upstreamRemote, ref); + checkoutChangelogs(gitWrapper, upstreamRemote, branch); Properties props = new Properties(); - props.load(new StringReader(gitWrapper.runCommand("git", "show", ref + ":build-tools-internal/version.properties"))); + props.load(new StringReader(gitWrapper.runCommand("git", "show", branch + ":build-tools-internal/version.properties"))); String version = props.getProperty("elasticsearch"); LOGGER.info("Finding changelog files..."); @@ -90,7 +119,27 @@ public void executeTask() throws IOException { .sorted(Comparator.comparing(ChangelogEntry::getPr)) .collect(toList()); - ChangelogBundle bundle = new ChangelogBundle(version, Instant.now().toString(), entries); + ChangelogBundle existingBundle = null; + + if (updateExisting) { + var existingBundleFile = new File("docs/release-notes/changelog-bundles/" + version + ".yml"); + if (existingBundleFile.exists()) { + var bundle = ChangelogBundle.parse(existingBundleFile); + existingBundle = bundle; + entries = entries.stream() + .filter(e -> bundle.changelogs().stream().anyMatch(c -> c.getPr().equals(e.getPr()))) + .toList(); + } + } + + var isReleased = false; + if (finalize) { + isReleased = true; + } else if (existingBundle != null) { + isReleased = existingBundle.released(); + } + + ChangelogBundle bundle = new ChangelogBundle(version, isReleased, Instant.now().toString(), entries); yamlMapper.writeValue(new File("docs/release-notes/changelog-bundles/" + version + ".yml"), bundle); } finally { @@ -102,7 +151,13 @@ private static void checkoutChangelogs(GitWrapper gitWrapper, String upstream, S gitWrapper.updateRemote(upstream); // TODO check for changes first gitWrapper.runCommand("rm", "-rf", "docs/changelog"); - gitWrapper.runCommand("git", "checkout", ref, "--", "docs/changelog"); + var refSpec = upstream + "/" + ref; + if (ref.contains("upstream/")) { + refSpec = ref.replace("upstream/", upstream + "/"); + } else if (ref.matches("^[0-9a-f]+$")) { + refSpec = ref; + } + gitWrapper.runCommand("git", "checkout", refSpec, "--", "docs/changelog"); } @InputDirectory diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java index b458770109353..355ea64355eb8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java @@ -21,13 +21,17 @@ import java.io.UncheckedIOException; import java.util.List; -public record ChangelogBundle(String version, String generated, List changelogs) { +public record ChangelogBundle(String version, boolean released, String generated, List changelogs) { private static final Logger LOGGER = Logging.getLogger(GenerateReleaseNotesTask.class); private static final ObjectMapper yamlMapper = new ObjectMapper( new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES).disable(YAMLGenerator.Feature.SPLIT_LINES) ); + public ChangelogBundle(String version, String generated, List changelogs) { + this(version, false, generated, changelogs); + } + public static ChangelogBundle parse(File file) { try { return yamlMapper.readValue(file, ChangelogBundle.class); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index e00491083dfc8..2d91fb18e9f2a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -97,15 +97,16 @@ public void executeTask() throws IOException { .getFiles() .stream() .map(ChangelogBundle::parse) - .sorted(Comparator.comparing(ChangelogBundle::generated).reversed()) + .sorted(Comparator.comparing(ChangelogBundle::released).reversed().thenComparing(ChangelogBundle::generated).reversed()) .toList(); // Ensure that each changelog/PR only shows up once, in its earliest release - // TODO: This should only be for unreleased/non-final bundles var uniquePrs = new HashSet(); for (int i = bundles.size() - 1; i >= 0; i--) { var bundle = bundles.get(i); - bundle.changelogs().removeAll(bundle.changelogs().stream().filter(c -> uniquePrs.contains(c.getPr())).toList()); + if (!bundle.released()) { + bundle.changelogs().removeAll(bundle.changelogs().stream().filter(c -> uniquePrs.contains(c.getPr())).toList()); + } uniquePrs.addAll(bundle.changelogs().stream().map(ChangelogEntry::getPr).toList()); } @@ -113,26 +114,6 @@ public void executeTask() throws IOException { ReleaseNotesGenerator.update(this.releaseNotesTemplate.get().getAsFile(), this.releaseNotesFile.get().getAsFile(), bundles); ReleaseNotesGenerator.update(this.breakingChangesTemplate.get().getAsFile(), this.breakingChangesFile.get().getAsFile(), bundles); ReleaseNotesGenerator.update(this.deprecationsTemplate.get().getAsFile(), this.deprecationsFile.get().getAsFile(), bundles); - - // Only update breaking changes and deprecations for new minors - // if (qualifiedVersion.revision() == 0) { - // LOGGER.info("Generating breaking changes / deprecations notes..."); - // ReleaseNotesGenerator.update( - // this.breakingChangesTemplate.get().getAsFile(), - // this.breakingChangesFile.get().getAsFile(), - // qualifiedVersion, - // changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()), - // bundles - // ); - // - // ReleaseNotesGenerator.update( - // this.deprecationsTemplate.get().getAsFile(), - // this.deprecationsFile.get().getAsFile(), - // qualifiedVersion, - // changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()), - // bundles - // ); - // } } /** diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java index 26236779a2390..283ba7457f9a8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java @@ -149,7 +149,7 @@ static Stream findPreviousVersion(GitWrapper gitWrapper, Quali final String currentMajorPattern = "v" + version.major() + ".*"; final String previousMajorPattern = "v" + (version.major() - 1) + ".*"; - return Stream.concat(gitWrapper.listVersions(currentMajorPattern), gitWrapper.listVersions(previousMajorPattern)) + return gitWrapper.listVersions(previousMajorPattern) .filter(v -> v.isBefore(version)); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index db05bb2d32cd1..a696d034be0bf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -73,6 +73,7 @@ public void apply(Project project) { task.setChangelogs(yamlFiles); task.setChangelogDirectory(changeLogDirectory); task.setBundleFile(projectDirectory.file("docs/release-notes/changelogs-" + version.toString() + ".yml")); + task.getOutputs().upToDateWhen(o -> false); }; final Function> configureGenerateTask = shouldConfigureYamlFiles -> task -> { @@ -98,6 +99,8 @@ public void apply(Project project) { task.setChangelogBundleDirectory(projectDirectory.dir("docs/release-notes/changelog-bundles")); + task.getOutputs().upToDateWhen(o -> false); + task.dependsOn(validateChangelogsTask); }; diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md index e2c9e463d87ff..04fa971b23cf0 100644 --- a/build-tools-internal/src/main/resources/templates/index.md +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -23,16 +23,24 @@ To check for security updates, go to [Security announcements for the Elastic sta <% for(bundle in changelogBundles) { def version = bundle.version - def versionWithoutSeparator = bundle.versionWithoutSeparator + def versionForIds = bundle.version.toString().equals('9.0.0') ? bundle.versionWithoutSeparator : bundle.version def changelogsByTypeByArea = bundle.changelogsByTypeByArea def notableHighlights = bundle.notableHighlights def nonNotableHighlights = bundle.nonNotableHighlights def unqualifiedVersion = bundle.unqualifiedVersion + def coming = !bundle.bundle.released + + if (coming) { + print "\n" + print "```{applies_to}\n" + print "stack: coming ${version}\n" + print "```" + } %> -## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-release-notes] +## ${unqualifiedVersion} [elasticsearch-${versionForIds}-release-notes] <% if (!notableHighlights.isEmpty() || !nonNotableHighlights.isEmpty()) { - print "\n### Highlights [elasticsearch-${versionWithoutSeparator}-highlights]\n" + print "\n### Highlights [elasticsearch-${versionForIds}-highlights]\n" } for (highlights in [notableHighlights, nonNotableHighlights]) { @@ -50,7 +58,7 @@ for (changeType in ['features-enhancements', 'fixes', 'regression']) { continue; } %> -### ${ TYPE_LABELS.getOrDefault(changeType, 'No mapping for TYPE_LABELS[' + changeType + ']') } [elasticsearch-${versionWithoutSeparator}-${changeType}] +### ${ TYPE_LABELS.getOrDefault(changeType, 'No mapping for TYPE_LABELS[' + changeType + ']') } [elasticsearch-${versionForIds}-${changeType}] <% for (team in changelogsByTypeByArea[changeType].keySet()) { print "\n${team}:\n"; diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index 543855a350331..b9e07a08a3dc3 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -10,8 +10,6 @@ Breaking changes can impact your Elastic applications, potentially disrupting no If you are migrating from a version prior to version 9.0, you must first upgrade to the last 8.x version available. To learn how to upgrade, check out [Upgrade](docs-content://deploy-manage/upgrade.md). -To learn how to upgrade, check out . - % ## Next version [elasticsearch-nextversion-breaking-changes] ## 9.0.1 [elasticsearch-901-breaking-changes] diff --git a/docs/release-notes/changelog-bundles/9.0.0.yml b/docs/release-notes/changelog-bundles/9.0.0.yml index 1766991b57b30..4b073d4fd8637 100644 --- a/docs/release-notes/changelog-bundles/9.0.0.yml +++ b/docs/release-notes/changelog-bundles/9.0.0.yml @@ -1,4 +1,5 @@ version: 9.0.0 +released: true generated: 2025-04-14T18:16:08.465456Z changelogs: - pr: 90529 diff --git a/docs/release-notes/changelog-bundles/9.0.1.yml b/docs/release-notes/changelog-bundles/9.0.1.yml index 806a1a5a97d4e..92394783847ec 100644 --- a/docs/release-notes/changelog-bundles/9.0.1.yml +++ b/docs/release-notes/changelog-bundles/9.0.1.yml @@ -1,11 +1,17 @@ version: 9.0.1 -generated: 2025-04-28T21:41:06.757953Z +released: false +generated: 2025-05-02T19:34:24.151692Z changelogs: - pr: 125694 summary: LTR score bounding area: Ranking type: bug issues: [] + - pr: 125922 + summary: Fix text structure NPE when fields in list have null value + area: Machine Learning + type: bug + issues: [] - pr: 126273 summary: Fix LTR rescorer with model alias area: Ranking @@ -130,3 +136,13 @@ changelogs: area: Ingest Node type: upgrade issues: [] + - pr: 127414 + summary: Fix npe when using source confirmed text query against missing field + area: Search + type: bug + issues: [] + - pr: 127527 + summary: "No, line noise isn't a valid ip" + area: ES|QL + type: bug + issues: [] diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 8fb66ab3754f0..b38ee35860e73 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -21,9 +21,12 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * -## 9.0.1 [elasticsearch-901-release-notes] +```{applies_to} +stack: coming 9.0.1 +``` +## 9.0.1 [elasticsearch-9.0.1-release-notes] -### Features and enhancements [elasticsearch-901-features-enhancements] +### Features and enhancements [elasticsearch-9.0.1-features-enhancements] Infra/Core: * Validation checks on paths allowed for 'files' entitlements. Restrict the paths we allow access to, forbidding plugins to specify/request entitlements for reading or writing to specific protected directories. [#126852](https://github.com/elastic/elasticsearch/pull/126852) @@ -37,7 +40,7 @@ Search: Security: * Add Issuer to failed SAML Signature validation logs when available [#126310](https://github.com/elastic/elasticsearch/pull/126310) (issue: [#111022](https://github.com/elastic/elasticsearch/issues/111022)) -### Fixes [elasticsearch-901-fixes] +### Fixes [elasticsearch-9.0.1-fixes] Aggregations: * Rare terms aggregation false **positive** fix [#126884](https://github.com/elastic/elasticsearch/pull/126884) @@ -55,6 +58,7 @@ ES|QL: * Fix count optimization with pushable union types [#127225](https://github.com/elastic/elasticsearch/pull/127225) (issue: [#127200](https://github.com/elastic/elasticsearch/issues/127200)) * Fix join masking eval [#126614](https://github.com/elastic/elasticsearch/pull/126614) * Fix sneaky bug in single value query [#127146](https://github.com/elastic/elasticsearch/pull/127146) +* No, line noise isn't a valid ip [#127527](https://github.com/elastic/elasticsearch/pull/127527) ILM+SLM: * Fix equality bug in `WaitForIndexColorStep` [#126605](https://github.com/elastic/elasticsearch/pull/126605) @@ -69,12 +73,16 @@ Infra/Core: Machine Learning: * Adding missing `onFailure` call for Inference API start model request [#126930](https://github.com/elastic/elasticsearch/pull/126930) +* Fix text structure NPE when fields in list have null value [#125922](https://github.com/elastic/elasticsearch/pull/125922) * Leverage threadpool schedule for inference api to avoid long running thread [#126858](https://github.com/elastic/elasticsearch/pull/126858) (issue: [#126853](https://github.com/elastic/elasticsearch/issues/126853)) Ranking: * Fix LTR rescorer with model alias [#126273](https://github.com/elastic/elasticsearch/pull/126273) * LTR score bounding [#125694](https://github.com/elastic/elasticsearch/pull/125694) +Search: +* Fix npe when using source confirmed text query against missing field [#127414](https://github.com/elastic/elasticsearch/pull/127414) + TSDB: * Improve resiliency of `UpdateTimeSeriesRangeService` [#126637](https://github.com/elastic/elasticsearch/pull/126637) From b5fa9e8a675347777f734efea704e45db85eeebb Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 2 May 2025 15:42:42 -0400 Subject: [PATCH 23/34] Fix the non-index pages --- .../src/main/resources/templates/breaking-changes.md | 12 ++++++++++-- .../src/main/resources/templates/deprecations.md | 12 ++++++++++-- docs/release-notes/breaking-changes.md | 5 ++++- docs/release-notes/deprecations.md | 5 ++++- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.md b/build-tools-internal/src/main/resources/templates/breaking-changes.md index b511e93471579..a4cbd8ce7c164 100644 --- a/build-tools-internal/src/main/resources/templates/breaking-changes.md +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.md @@ -14,11 +14,19 @@ If you are migrating from a version prior to version 9.0, you must first upgrade <% for(bundle in changelogBundles) { def version = bundle.version - def versionWithoutSeparator = bundle.versionWithoutSeparator + def versionForIds = bundle.version.toString().equals('9.0.0') ? bundle.versionWithoutSeparator : bundle.version def changelogsByTypeByArea = bundle.changelogsByTypeByArea def unqualifiedVersion = bundle.unqualifiedVersion + def coming = !bundle.bundle.released + + if (coming) { + print "\n" + print "```{applies_to}\n" + print "stack: coming ${version}\n" + print "```" + } %> -## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-breaking-changes] +## ${unqualifiedVersion} [elasticsearch-${versionForIds}-breaking-changes] <% if (!changelogsByTypeByArea['breaking']) { print "\nNo breaking changes in this version.\n" diff --git a/build-tools-internal/src/main/resources/templates/deprecations.md b/build-tools-internal/src/main/resources/templates/deprecations.md index 795ff81d9f56a..38c58682a999b 100644 --- a/build-tools-internal/src/main/resources/templates/deprecations.md +++ b/build-tools-internal/src/main/resources/templates/deprecations.md @@ -18,11 +18,19 @@ To give you insight into what deprecated features you’re using, {{es}}: <% for(bundle in changelogBundles) { def version = bundle.version - def versionWithoutSeparator = bundle.versionWithoutSeparator + def versionForIds = bundle.version.toString().equals('9.0.0') ? bundle.versionWithoutSeparator : bundle.version def changelogsByTypeByArea = bundle.changelogsByTypeByArea def unqualifiedVersion = bundle.unqualifiedVersion + def coming = !bundle.bundle.released + + if (coming) { + print "\n" + print "```{applies_to}\n" + print "stack: coming ${version}\n" + print "```" + } %> -## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-deprecations] +## ${unqualifiedVersion} [elasticsearch-${versionForIds}-deprecations] <% if (!changelogsByTypeByArea['deprecation']) { print "\nNo deprecations in this version.\n" diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index b9e07a08a3dc3..dda79d1e308b6 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -12,7 +12,10 @@ If you are migrating from a version prior to version 9.0, you must first upgrade % ## Next version [elasticsearch-nextversion-breaking-changes] -## 9.0.1 [elasticsearch-901-breaking-changes] +```{applies_to} +stack: coming 9.0.1 +``` +## 9.0.1 [elasticsearch-9.0.1-breaking-changes] No breaking changes in this version. diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index 14a3215381c83..1cb9fe15363b7 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -16,7 +16,10 @@ To give you insight into what deprecated features you’re using, {{es}}: % ## Next version [elasticsearch-nextversion-deprecations] -## 9.0.1 [elasticsearch-901-deprecations] +```{applies_to} +stack: coming 9.0.1 +``` +## 9.0.1 [elasticsearch-9.0.1-deprecations] No deprecations in this version. From 0141be1d7bbae98c197b24cbde773f408a028922 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 2 May 2025 19:50:15 +0000 Subject: [PATCH 24/34] [CI] Auto commit changes from spotless --- .../gradle/internal/release/BundleChangelogsTask.java | 1 - .../gradle/internal/release/PruneChangelogsTask.java | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index a6013b720cb5a..c693bd42784b7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -14,7 +14,6 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; -import org.elasticsearch.gradle.Version; import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.Directory; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java index 283ba7457f9a8..489b3561150fc 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java @@ -149,8 +149,7 @@ static Stream findPreviousVersion(GitWrapper gitWrapper, Quali final String currentMajorPattern = "v" + version.major() + ".*"; final String previousMajorPattern = "v" + (version.major() - 1) + ".*"; - return gitWrapper.listVersions(previousMajorPattern) - .filter(v -> v.isBefore(version)); + return gitWrapper.listVersions(previousMajorPattern).filter(v -> v.isBefore(version)); } /** From 71d5879189b378510e98e3d9a775c445abfd0849 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 5 May 2025 16:52:39 -0400 Subject: [PATCH 25/34] Fix template and add --bc-ref support --- .../release/BundleChangelogsTask.java | 107 ++++++++++++------ .../src/main/resources/templates/index.md | 1 - 2 files changed, 75 insertions(+), 33 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index a6013b720cb5a..5c60d19021b2f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -39,6 +39,8 @@ import java.util.Comparator; import java.util.List; import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nullable; import javax.inject.Inject; @@ -57,22 +59,34 @@ public class BundleChangelogsTask extends DefaultTask { @Nullable private String branch; - private boolean updateExisting; + @Nullable + private String bcRef; + // private boolean updateExisting; private boolean finalize; - @Option( - option = "update-existing", - description = "Only update entries that are already in the bundle. Useful for updating the bundle after a BC has been cut." - ) - public void setUpdateExisting(boolean updateExisting) { - this.updateExisting = updateExisting; - } + // @Option( + // option = "update-existing", + // description = "Only update entries that are already in the bundle. Useful for updating the bundle after a BC has been cut." + // ) + // public void setUpdateExisting(boolean updateExisting) { + // this.updateExisting = updateExisting; + // } @Option(option = "branch", description = "Branch (or other ref) to use for generating the changelog bundle.") public void setBranch(String branch) { this.branch = branch; } + @Option( + option = "bc-ref", + description = "A source ref, typically the sha of a BC, that should be used to source PRs for changelog entries. " + + "The actual content of the changelogs will come from the 'branch' ref. " + + "You should generally always use bc-ref." + ) + public void setBcRef(String ref) { + this.bcRef = ref; + } + @Option(option = "finalize", description = "Specify that the bundle is finalized, i.e. that the version has been released.") public void setFinalize(boolean finalize) { this.finalize = finalize; @@ -103,41 +117,59 @@ public void executeTask() throws IOException { } final String upstreamRemote = gitWrapper.getUpstream(); + Set entriesFromBc = Set.of(); try { - checkoutChangelogs(gitWrapper, upstreamRemote, branch); + var usingBcRef = bcRef != null && !bcRef.isEmpty(); + if (usingBcRef) { + // Check out all the changelogs that existed at the time of the BC + checkoutChangelogs(gitWrapper, upstreamRemote, bcRef); + entriesFromBc = this.changelogDirectory.getAsFileTree().getFiles().stream().map(File::getName).collect(Collectors.toSet()); + + // Then add/update changelogs from the HEAD of the branch + // We do an "add" here, rather than checking out the entire directory, in case changelogs have been removed for some reason + addChangelogsFromRef(gitWrapper, upstreamRemote, branch); + } else { + checkoutChangelogs(gitWrapper, upstreamRemote, branch); + } Properties props = new Properties(); props.load(new StringReader(gitWrapper.runCommand("git", "show", branch + ":build-tools-internal/version.properties"))); String version = props.getProperty("elasticsearch"); LOGGER.info("Finding changelog files..."); - List entries = this.changelogDirectory.getAsFileTree() - .getFiles() - .stream() - .map(ChangelogEntry::parse) - .sorted(Comparator.comparing(ChangelogEntry::getPr)) - .collect(toList()); - - ChangelogBundle existingBundle = null; + Set finalEntriesFromBc = entriesFromBc; + List entries = this.changelogDirectory.getAsFileTree().getFiles().stream().filter(f -> { + // When not using a bc ref, we just take everything from the branch/sha passed in + if (!usingBcRef) { + return true; + } - if (updateExisting) { - var existingBundleFile = new File("docs/release-notes/changelog-bundles/" + version + ".yml"); - if (existingBundleFile.exists()) { - var bundle = ChangelogBundle.parse(existingBundleFile); - existingBundle = bundle; - entries = entries.stream() - .filter(e -> bundle.changelogs().stream().anyMatch(c -> c.getPr().equals(e.getPr()))) - .toList(); + // If the changelog was present in the BC sha, use it + if (finalEntriesFromBc.contains(f.getName())) { + return true; } - } - var isReleased = false; - if (finalize) { - isReleased = true; - } else if (existingBundle != null) { - isReleased = existingBundle.released(); - } + // Otherwise, let's check to see if a reference to the PR exists in the commit log for the sha + // This specifically covers the case of a PR being merged into the BC with a missing changelog file, and the file added + // later. + var prNumber = f.getName().replace(".yaml", ""); + return !gitWrapper.runCommand("git", "log", bcRef, "--grep", "(#" + prNumber + ")").trim().isEmpty(); + }).map(ChangelogEntry::parse).sorted(Comparator.comparing(ChangelogEntry::getPr)).collect(toList()); + + ChangelogBundle existingBundle = null; + // if (updateExisting) { + // var existingBundleFile = new File("docs/release-notes/changelog-bundles/" + version + ".yml"); + // if (existingBundleFile.exists()) { + // var bundle = ChangelogBundle.parse(existingBundleFile); + // existingBundle = bundle; + // entries = entries.stream() + // .filter(e -> bundle.changelogs().stream().anyMatch(c -> c.getPr().equals(e.getPr()))) + // .toList(); + // } + // } + + var isReleased = finalize == true; ChangelogBundle bundle = new ChangelogBundle(version, isReleased, Instant.now().toString(), entries); @@ -160,6 +192,17 @@ private static void checkoutChangelogs(GitWrapper gitWrapper, String upstream, S gitWrapper.runCommand("git", "checkout", refSpec, "--", "docs/changelog"); } + private static void addChangelogsFromRef(GitWrapper gitWrapper, String upstream, String ref) { + var refSpec = upstream + "/" + ref; + if (ref.contains("upstream/")) { + refSpec = ref.replace("upstream/", upstream + "/"); + } else if (ref.matches("^[0-9a-f]+$")) { + refSpec = ref; + } + + gitWrapper.runCommand("git", "checkout", refSpec, "--", "docs/changelog/*.yaml"); + } + @InputDirectory public DirectoryProperty getChangelogDirectory() { return changelogDirectory; diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md index 04fa971b23cf0..cb417eb49dc64 100644 --- a/build-tools-internal/src/main/resources/templates/index.md +++ b/build-tools-internal/src/main/resources/templates/index.md @@ -1,7 +1,6 @@ --- navigation_title: "Elasticsearch" mapped_pages: - - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html --- From 580f37d968ceb2057eee395d034c0062ae87a1b7 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 6 May 2025 13:20:38 -0400 Subject: [PATCH 26/34] Formatting --- .../gradle/internal/release/BundleChangelogsTask.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index 9df0faaeb7262..935cf6e3bb8a3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -119,7 +119,7 @@ public void executeTask() throws IOException { Set entriesFromBc = Set.of(); try { - var usingBcRef = bcRef != null && !bcRef.isEmpty(); + var usingBcRef = bcRef != null && bcRef.isEmpty() == false; if (usingBcRef) { // Check out all the changelogs that existed at the time of the BC checkoutChangelogs(gitWrapper, upstreamRemote, bcRef); @@ -140,7 +140,7 @@ public void executeTask() throws IOException { Set finalEntriesFromBc = entriesFromBc; List entries = this.changelogDirectory.getAsFileTree().getFiles().stream().filter(f -> { // When not using a bc ref, we just take everything from the branch/sha passed in - if (!usingBcRef) { + if (usingBcRef == false) { return true; } @@ -153,7 +153,8 @@ public void executeTask() throws IOException { // This specifically covers the case of a PR being merged into the BC with a missing changelog file, and the file added // later. var prNumber = f.getName().replace(".yaml", ""); - return !gitWrapper.runCommand("git", "log", bcRef, "--grep", "(#" + prNumber + ")").trim().isEmpty(); + var output = gitWrapper.runCommand("git", "log", bcRef, "--grep", "(#" + prNumber + ")"); + return output.trim().isEmpty() == false; }).map(ChangelogEntry::parse).sorted(Comparator.comparing(ChangelogEntry::getPr)).collect(toList()); ChangelogBundle existingBundle = null; From 5413020ee7feaf607cb5ddd69d8f016f54b8930f Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 6 May 2025 15:43:03 -0400 Subject: [PATCH 27/34] Finalize 9.0.1 release notes --- docs/release-notes/breaking-changes.md | 3 --- docs/release-notes/changelog-bundles/9.0.1.yml | 4 ++-- docs/release-notes/deprecations.md | 3 --- docs/release-notes/index.md | 4 ---- 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index dda79d1e308b6..875211b52944b 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -12,9 +12,6 @@ If you are migrating from a version prior to version 9.0, you must first upgrade % ## Next version [elasticsearch-nextversion-breaking-changes] -```{applies_to} -stack: coming 9.0.1 -``` ## 9.0.1 [elasticsearch-9.0.1-breaking-changes] No breaking changes in this version. diff --git a/docs/release-notes/changelog-bundles/9.0.1.yml b/docs/release-notes/changelog-bundles/9.0.1.yml index 92394783847ec..2ed6fda90ff7b 100644 --- a/docs/release-notes/changelog-bundles/9.0.1.yml +++ b/docs/release-notes/changelog-bundles/9.0.1.yml @@ -1,6 +1,6 @@ version: 9.0.1 -released: false -generated: 2025-05-02T19:34:24.151692Z +released: true +generated: 2025-05-06T19:39:27.474912Z changelogs: - pr: 125694 summary: LTR score bounding diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index 1cb9fe15363b7..a691d9e28a1e2 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -16,9 +16,6 @@ To give you insight into what deprecated features you’re using, {{es}}: % ## Next version [elasticsearch-nextversion-deprecations] -```{applies_to} -stack: coming 9.0.1 -``` ## 9.0.1 [elasticsearch-9.0.1-deprecations] No deprecations in this version. diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index b38ee35860e73..4e841acf18ba3 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -1,7 +1,6 @@ --- navigation_title: "Elasticsearch" mapped_pages: - - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html --- @@ -21,9 +20,6 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * -```{applies_to} -stack: coming 9.0.1 -``` ## 9.0.1 [elasticsearch-9.0.1-release-notes] ### Features and enhancements [elasticsearch-9.0.1-features-enhancements] From 87ede2f335227de9e60f36d2562d42ef9e4f98b4 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 19 May 2025 16:58:26 -0400 Subject: [PATCH 28/34] Tests for release note generation via bundles --- .../internal/release/ChangelogBundle.java | 9 + .../release/GenerateReleaseNotesTask.java | 61 ++-- .../release/GenerateReleaseNotesTaskTest.java | 272 ------------------ .../release/ReleaseNotesGeneratorTest.java | 52 ++-- ...easeNotesGeneratorTest.breaking-changes.md | 65 ++++- .../ReleaseNotesGeneratorTest.deprecations.md | 65 ++++- .../ReleaseNotesGeneratorTest.index.md | 131 ++++++--- ...easeNotesGeneratorTest.index.no-changes.md | 9 - ...eNotesGeneratorTest.index.no-highlights.md | 97 +++++-- 9 files changed, 331 insertions(+), 430 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java index 355ea64355eb8..68e08d9769188 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java @@ -40,4 +40,13 @@ public static ChangelogBundle parse(File file) { throw new UncheckedIOException(e); } } + + public static ChangelogBundle copy(ChangelogBundle bundle) { + List changelogs = bundle.changelogs().stream().toList(); + return new ChangelogBundle(bundle.version(), bundle.released(), bundle.generated(), changelogs); + } + + public ChangelogBundle withChangelogs(List changelogs) { + return new ChangelogBundle(version, released, generated, changelogs); + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index 2d91fb18e9f2a..62ebbdf99bd49 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -31,6 +31,7 @@ import org.gradle.process.ExecOperations; import java.io.IOException; +import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -93,22 +94,13 @@ public void executeTask() throws IOException { LOGGER.info("Finding changelog bundles..."); - List bundles = this.changelogBundleDirectory.getAsFileTree() + List allBundles = this.changelogBundleDirectory.getAsFileTree() .getFiles() .stream() .map(ChangelogBundle::parse) - .sorted(Comparator.comparing(ChangelogBundle::released).reversed().thenComparing(ChangelogBundle::generated).reversed()) .toList(); - // Ensure that each changelog/PR only shows up once, in its earliest release - var uniquePrs = new HashSet(); - for (int i = bundles.size() - 1; i >= 0; i--) { - var bundle = bundles.get(i); - if (!bundle.released()) { - bundle.changelogs().removeAll(bundle.changelogs().stream().filter(c -> uniquePrs.contains(c.getPr())).toList()); - } - uniquePrs.addAll(bundle.changelogs().stream().map(ChangelogEntry::getPr).toList()); - } + var bundles = getSortedBundlesWithUniqueChangelogs(allBundles); LOGGER.info("Generating release notes..."); ReleaseNotesGenerator.update(this.releaseNotesTemplate.get().getAsFile(), this.releaseNotesFile.get().getAsFile(), bundles); @@ -116,37 +108,28 @@ public void executeTask() throws IOException { ReleaseNotesGenerator.update(this.deprecationsTemplate.get().getAsFile(), this.deprecationsFile.get().getAsFile(), bundles); } - /** - * Find all tags in the major series for the supplied version - * @param gitWrapper used to call `git` - * @param currentVersion the version to base the query upon - * @return all versions in the series - */ @VisibleForTesting - static Set getVersions(GitWrapper gitWrapper, String currentVersion) { - QualifiedVersion qualifiedVersion = QualifiedVersion.of(currentVersion); - final String pattern = "v" + qualifiedVersion.major() + ".*"; - // We may be generating notes for a minor version prior to the latest minor, so we need to filter out versions that are too new. - Set versions = Stream.concat( - gitWrapper.listVersions(pattern).filter(v -> v.isBefore(qualifiedVersion)), - Stream.of(qualifiedVersion) - ).collect(toSet()); - - // If this is a new minor ensure we include the previous minor, which may not have been released - if (qualifiedVersion.minor() > 0 && qualifiedVersion.revision() == 0) { - QualifiedVersion previousMinor = new QualifiedVersion(qualifiedVersion.major(), qualifiedVersion.minor() - 1, 0, null); - versions.add(previousMinor); - } + static List getSortedBundlesWithUniqueChangelogs(List bundles) { + List sorted = bundles.stream() + // .map(ChangelogBundle::copy) + .sorted(Comparator.comparing(ChangelogBundle::released).reversed().thenComparing(ChangelogBundle::generated)) + .toList(); - return versions; - } + // Ensure that each changelog/PR only shows up once, in its earliest release + var uniquePrs = new HashSet(); + List modifiedBundles = new ArrayList<>(); + for (int i = sorted.size() - 1; i >= 0; i--) { + var bundle = sorted.get(i); + if (bundle.released() == false) { + List entries = bundle.changelogs().stream().filter(c -> false == uniquePrs.contains(c.getPr())).toList(); + modifiedBundles.add(bundle.withChangelogs(entries)); + } else { + modifiedBundles.add(bundle); + } + uniquePrs.addAll(bundle.changelogs().stream().map(ChangelogEntry::getPr).toList()); + } - /** - * Convert set of QualifiedVersion to MinorVersion by deleting all but the major and minor components. - */ - @VisibleForTesting - static Set getMinorVersions(Set versions) { - return versions.stream().map(MinorVersion::of).collect(toSet()); + return modifiedBundles; } /** diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTaskTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTaskTest.java index 78bdb8017f476..98b09bbcaf4dc 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTaskTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTaskTest.java @@ -9,38 +9,12 @@ package org.elasticsearch.gradle.internal.release; -import org.junit.Before; import org.junit.Test; -import java.io.File; -import java.util.Map; -import java.util.Set; -import java.util.stream.Stream; - import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; public class GenerateReleaseNotesTaskTest { - private GitWrapper gitWrapper; - - @Before - public void setup() { - this.gitWrapper = mock(GitWrapper.class); - } - /** * Check that the task does not update git tags if the current version is a snapshot of the first patch release. */ @@ -88,250 +62,4 @@ public void needsGitTags_withFirsAlphaRelease_returnsFalse() { public void needsGitTags_withLaterAlphaRelease_returnsFalse() { assertThat(GenerateReleaseNotesTask.needsGitTags("8.0.0-alpha2"), is(true)); } - - /** - * Check that partitioning changelog files when the current version is a snapshot returns a map with a single entry. - */ - @Test - public void partitionFiles_withSnapshot_returnsSingleMapping() { - // when: - Map> partitionedFiles = GenerateReleaseNotesTask.partitionFilesByVersion( - gitWrapper, - "8.0.0-SNAPSHOT", - Set.of(new File("docs/changelog/1234.yaml")) - ); - - // then: - assertThat(partitionedFiles, aMapWithSize(1)); - assertThat( - partitionedFiles, - hasEntry(equalTo(QualifiedVersion.of("8.0.0-SNAPSHOT")), hasItem(new File("docs/changelog/1234.yaml"))) - ); - verifyNoMoreInteractions(gitWrapper); - } - - /** - * Check that partitioning changelog files when the current version is the first release - * in a minor series returns a map with a single entry. - */ - @Test - public void partitionFiles_withFirstRevision_returnsSingleMapping() { - // when: - Map> partitionedFiles = GenerateReleaseNotesTask.partitionFilesByVersion( - gitWrapper, - "8.5.0", - Set.of(new File("docs/changelog/1234.yaml")) - ); - - // then: - assertThat(partitionedFiles, aMapWithSize(1)); - assertThat(partitionedFiles, hasEntry(equalTo(QualifiedVersion.of("8.5.0")), hasItem(new File("docs/changelog/1234.yaml")))); - verifyNoMoreInteractions(gitWrapper); - } - - /** - * Check that partitioning changelog files when the current version is the first alpha prerelease returns a map with a single entry. - */ - @Test - public void partitionFiles_withFirstAlpha_returnsSingleMapping() { - // when: - Map> partitionedFiles = GenerateReleaseNotesTask.partitionFilesByVersion( - gitWrapper, - "8.0.0-alpha1", - Set.of(new File("docs/changelog/1234.yaml")) - ); - - // then: - assertThat(partitionedFiles, aMapWithSize(1)); - assertThat(partitionedFiles, hasEntry(equalTo(QualifiedVersion.of("8.0.0-alpha1")), hasItem(new File("docs/changelog/1234.yaml")))); - verifyNoMoreInteractions(gitWrapper); - } - - /** - * Check that when deriving a lit of versions from git tags, the current unreleased version is included. - */ - @Test - public void getVersions_includesCurrentAndPreviousVersion() { - // given: - when(gitWrapper.listVersions(anyString())).thenReturn( - Stream.of("8.0.0-alpha1", "8.0.0-alpha2", "8.0.0-beta1", "8.0.0-beta2", "8.0.0-beta3", "8.0.0-rc1", "8.0.0", "8.0.1", "8.1.0") - .map(QualifiedVersion::of) - ); - - // when: - Set versions = GenerateReleaseNotesTask.getVersions(gitWrapper, "8.3.0-SNAPSHOT"); - - // then: - assertThat( - versions, - containsInAnyOrder( - Stream.of( - "8.0.0-alpha1", - "8.0.0-alpha2", - "8.0.0-beta1", - "8.0.0-beta2", - "8.0.0-beta3", - "8.0.0-rc1", - "8.0.0", - "8.0.1", - "8.1.0", - "8.2.0", - "8.3.0-SNAPSHOT" - ).map(QualifiedVersion::of).toArray(QualifiedVersion[]::new) - ) - ); - } - - /** - * Check that when deriving a list of major.minor versions from git tags, the current unreleased version is included, - * but any higher version numbers are not. - */ - @Test - public void getMinorVersions_includesCurrentButNotFutureVersions() { - // given: - when(gitWrapper.listVersions(anyString())).thenReturn( - Stream.of("8.0.0-alpha1", "8.0.0-alpha2", "8.0.0", "8.0.1", "8.1.0", "8.2.0", "8.2.1", "8.3.0", "8.3.1", "8.4.0") - .map(QualifiedVersion::of) - ); - - // when: - Set versions = GenerateReleaseNotesTask.getVersions(gitWrapper, "8.3.0-SNAPSHOT"); - Set minorVersions = GenerateReleaseNotesTask.getMinorVersions(versions); - - // then: - assertThat( - minorVersions, - containsInAnyOrder(new MinorVersion(8, 0), new MinorVersion(8, 1), new MinorVersion(8, 2), new MinorVersion(8, 3)) - ); - } - - /** - * Check that the task partitions the list of files correctly by version for a prerelease. - */ - @Test - public void partitionFiles_withPrerelease_correctlyGroupsByPrereleaseVersion() { - // given: - when(gitWrapper.listVersions(anyString())).thenReturn( - Stream.of("8.0.0-alpha1", "8.0.0-alpha2", "8.0.0-beta1", "8.0.0-beta2", "8.0.0-beta3", "8.0.0-rc1", "8.0.0") - .map(QualifiedVersion::of) - ); - when(gitWrapper.listFiles(eq("v8.0.0-alpha1"), anyString())).thenReturn( - Stream.of("docs/changelog/1_1234.yaml", "docs/changelog/1_5678.yaml") - ); - when(gitWrapper.listFiles(eq("v8.0.0-alpha2"), anyString())).thenReturn( - Stream.of("docs/changelog/2_1234.yaml", "docs/changelog/2_5678.yaml") - ); - - Set allFiles = Set.of( - new File("docs/changelog/1_1234.yaml"), - new File("docs/changelog/1_5678.yaml"), - new File("docs/changelog/2_1234.yaml"), - new File("docs/changelog/2_5678.yaml"), - new File("docs/changelog/3_1234.yaml"), - new File("docs/changelog/3_5678.yaml") - ); - - // when: - Map> partitionedFiles = GenerateReleaseNotesTask.partitionFilesByVersion( - gitWrapper, - "8.0.0-beta1", - allFiles - ); - - // then: - verify(gitWrapper).listVersions("v8.0*"); - verify(gitWrapper).listFiles("v8.0.0-alpha1", "docs/changelog"); - verify(gitWrapper).listFiles("v8.0.0-alpha2", "docs/changelog"); - - assertThat( - partitionedFiles, - allOf( - aMapWithSize(3), - hasKey(QualifiedVersion.of("8.0.0-alpha1")), - hasKey(QualifiedVersion.of("8.0.0-alpha2")), - hasKey(QualifiedVersion.of("8.0.0-beta1")) - ) - ); - - assertThat( - partitionedFiles, - allOf( - hasEntry( - equalTo(QualifiedVersion.of("8.0.0-alpha1")), - containsInAnyOrder(new File("docs/changelog/1_1234.yaml"), new File("docs/changelog/1_5678.yaml")) - ), - hasEntry( - equalTo(QualifiedVersion.of("8.0.0-alpha2")), - containsInAnyOrder(new File("docs/changelog/2_1234.yaml"), new File("docs/changelog/2_5678.yaml")) - ), - hasEntry( - equalTo(QualifiedVersion.of("8.0.0-beta1")), - containsInAnyOrder(new File("docs/changelog/3_1234.yaml"), new File("docs/changelog/3_5678.yaml")) - ) - ) - ); - } - - /** - * Check that the task partitions the list of files correctly by version for a patch release. - */ - @Test - public void partitionFiles_withPatchRelease_correctlyGroupsByPatchVersion() { - // given: - when(gitWrapper.listVersions(anyString())).thenReturn( - Stream.of("8.0.0-alpha1", "8.0.0-alpha2", "8.0.0-beta1", "8.0.0-rc1", "8.0.0", "8.0.1", "8.0.2", "8.1.0") - .map(QualifiedVersion::of) - ); - when(gitWrapper.listFiles(eq("v8.0.0"), anyString())).thenReturn( - Stream.of("docs/changelog/1_1234.yaml", "docs/changelog/1_5678.yaml") - ); - when(gitWrapper.listFiles(eq("v8.0.1"), anyString())).thenReturn( - Stream.of("docs/changelog/2_1234.yaml", "docs/changelog/2_5678.yaml") - ); - - Set allFiles = Set.of( - new File("docs/changelog/1_1234.yaml"), - new File("docs/changelog/1_5678.yaml"), - new File("docs/changelog/2_1234.yaml"), - new File("docs/changelog/2_5678.yaml"), - new File("docs/changelog/3_1234.yaml"), - new File("docs/changelog/3_5678.yaml") - ); - - // when: - Map> partitionedFiles = GenerateReleaseNotesTask.partitionFilesByVersion(gitWrapper, "8.0.2", allFiles); - - // then: - verify(gitWrapper).listVersions("v8.0*"); - verify(gitWrapper).listFiles("v8.0.0", "docs/changelog"); - verify(gitWrapper).listFiles("v8.0.1", "docs/changelog"); - - assertThat( - partitionedFiles, - allOf( - aMapWithSize(3), - hasKey(QualifiedVersion.of("8.0.0")), - hasKey(QualifiedVersion.of("8.0.1")), - hasKey(QualifiedVersion.of("8.0.2")) - ) - ); - - assertThat( - partitionedFiles, - allOf( - hasEntry( - equalTo(QualifiedVersion.of("8.0.0")), - containsInAnyOrder(new File("docs/changelog/1_1234.yaml"), new File("docs/changelog/1_5678.yaml")) - ), - hasEntry( - equalTo(QualifiedVersion.of("8.0.1")), - containsInAnyOrder(new File("docs/changelog/2_1234.yaml"), new File("docs/changelog/2_5678.yaml")) - ), - hasEntry( - equalTo(QualifiedVersion.of("8.0.2")), - containsInAnyOrder(new File("docs/changelog/3_1234.yaml"), new File("docs/changelog/3_5678.yaml")) - ) - ) - ); - } } diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java index 96b8a32a0745f..b57b9feb137a0 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java @@ -13,14 +13,13 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; +import static org.elasticsearch.gradle.internal.release.GenerateReleaseNotesTask.getSortedBundlesWithUniqueChangelogs; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThat; @@ -47,17 +46,17 @@ public void generateFile_index_rendersCorrectMarkup() throws Exception { @Test public void generateFile_index_noHighlights_rendersCorrectMarkup() throws Exception { - Set entries = getEntries(); - entries = entries.stream().filter(e -> e.getHighlight() == null).collect(Collectors.toSet()); + var bundles = getBundles(); + bundles = bundles.stream().filter(b -> false == b.version().equals("9.1.0")).toList(); - testTemplate("index.md", "index.no-highlights.md", entries); + testTemplate("index.md", "index.no-highlights.md", bundles); } @Test public void generateFile_index_noChanges_rendersCorrectMarkup() throws Exception { - Set entries = new HashSet<>(); + var bundles = new ArrayList(); - testTemplate("index.md", "index.no-changes.md", entries); + testTemplate("index.md", "index.no-changes.md", bundles); } @Test @@ -78,33 +77,48 @@ public void testTemplate(String templateFilename, String outputFilename) throws testTemplate(templateFilename, outputFilename, null); } - public void testTemplate(String templateFilename, String outputFilename, Set entries) throws Exception { + public void testTemplate(String templateFilename, String outputFilename, List bundles) throws Exception { // given: final String template = getResource("/templates/" + templateFilename); final String expectedOutput = getResource("/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest." + outputFilename); - if (entries == null) { - entries = getEntries(); + if (bundles == null) { + bundles = getBundles(); } + bundles = getSortedBundlesWithUniqueChangelogs(bundles); + // when: - final String actualOutput = ReleaseNotesGenerator.generateFile(template, QualifiedVersion.of("8.2.0-SNAPSHOT"), entries); + final String actualOutput = ReleaseNotesGenerator.generateFile(template, bundles); + + Files.write(Path.of("/Users/bseeders/output.txt"), actualOutput.getBytes()); // then: assertThat(actualOutput, equalTo(expectedOutput)); } - private Set getEntries() { - final Set entries = new HashSet<>(); + private List getBundles() { + List bundles = new ArrayList<>(); + for (int i = 0; i < CHANGE_TYPES.size(); i++) { - entries.addAll(buildEntries(i, 2)); + bundles.add( + new ChangelogBundle( + "9.0." + i, + i != CHANGE_TYPES.size() - 1, + "2025-05-16T00:00:" + String.format("%d", 10 + i), + buildEntries(i, 2) + ) + ); } - entries.add(makeHighlightsEntry(5001, false)); - entries.add(makeHighlightsEntry(5000, true)); - entries.add(makeHighlightsEntry(5002, true)); + final List entries = new ArrayList<>(); + entries.add(makeHighlightsEntry(51, false)); + entries.add(makeHighlightsEntry(50, true)); + entries.add(makeHighlightsEntry(52, true)); - return entries; + bundles.add(new ChangelogBundle("9.1.0", false, "2025-05-17T00:00:00", entries)); + + return bundles; } private List buildEntries(int seed, int count) { diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md index 038521f311242..1b8d1f3a981cb 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.breaking-changes.md @@ -1,22 +1,71 @@ --- -navigation_title: "Elasticsearch" +navigation_title: "Breaking changes" mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes.html --- # Elasticsearch breaking changes [elasticsearch-breaking-changes] -Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. -To learn how to upgrade, check out . +Breaking changes can impact your Elastic applications, potentially disrupting normal operations. Before you upgrade, carefully review the Elasticsearch breaking changes and take the necessary steps to mitigate any issues. + +If you are migrating from a version prior to version 9.0, you must first upgrade to the last 8.x version available. To learn how to upgrade, check out [Upgrade](docs-content://deploy-manage/upgrade.md). % ## Next version [elasticsearch-nextversion-breaking-changes] -% **Release date:** Month day, year -## 8.2.0 [elasticsearch-820-breaking-changes] -**Release date:** April 01, 2025 +```{applies_to} +stack: coming 9.1.0 +``` +## 9.1.0 [elasticsearch-9.1.0-breaking-changes] + +No breaking changes in this version. + +```{applies_to} +stack: coming 9.0.10 +``` +## 9.0.10 [elasticsearch-9.0.10-breaking-changes] + +No breaking changes in this version. + +## 9.0.9 [elasticsearch-9.0.9-breaking-changes] + +No breaking changes in this version. + +## 9.0.8 [elasticsearch-9.0.8-breaking-changes] + +No breaking changes in this version. + +## 9.0.7 [elasticsearch-9.0.7-breaking-changes] + +No breaking changes in this version. + +## 9.0.6 [elasticsearch-9.0.6-breaking-changes] + +No breaking changes in this version. + +## 9.0.5 [elasticsearch-9.0.5-breaking-changes] + +No breaking changes in this version. + +## 9.0.4 [elasticsearch-9.0.4-breaking-changes] + +No breaking changes in this version. + +## 9.0.3 [elasticsearch-9.0.3-breaking-changes] + +No breaking changes in this version. + +## 9.0.2 [elasticsearch-9.0.2-breaking-changes] + +No breaking changes in this version. + +## 9.0.1 [elasticsearch-9.0.1-breaking-changes] + +No breaking changes in this version. + +## 9.0.0 [elasticsearch-900-breaking-changes] Aggregation: -* Test changelog entry 0_0 [#0](https://github.com/elastic/elasticsearch/pull/0) (issue: {es-issue}1[#1]) -* Test changelog entry 0_1 [#2](https://github.com/elastic/elasticsearch/pull/2) (issues: {es-issue}3[#3], {es-issue}4[#4]) +* Test changelog entry 0_0 [#0](https://github.com/elastic/elasticsearch/pull/0) (issue: [#1](https://github.com/elastic/elasticsearch/issues/1)) +* Test changelog entry 0_1 [#2](https://github.com/elastic/elasticsearch/pull/2) (issues: [#3](https://github.com/elastic/elasticsearch/issues/3), [#4](https://github.com/elastic/elasticsearch/issues/4)) diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md index 6d36fc53181e7..35587f559014b 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.deprecations.md @@ -1,11 +1,12 @@ --- -navigation_title: "Elasticsearch" +navigation_title: "Deprecations" --- # {{es}} deprecations [elasticsearch-deprecations] -Review the deprecated functionality for your {{es}} version. While deprecations have no immediate impact, we strongly encourage you update your implementation after you upgrade. -To learn how to upgrade, check out . +Over time, certain Elastic functionality becomes outdated and is replaced or removed. To help with the transition, Elastic deprecates functionality for a period before removal, giving you time to update your applications. + +Review the deprecated functionality for Elasticsearch. While deprecations have no immediate impact, we strongly encourage you update your implementation after you upgrade. To learn how to upgrade, check out [Upgrade](docs-content://deploy-manage/upgrade.md). To give you insight into what deprecated features you’re using, {{es}}: @@ -14,13 +15,61 @@ To give you insight into what deprecated features you’re using, {{es}}: * [Provides a deprecation info API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-migration-deprecations) that scans a cluster’s configuration and mappings for deprecated functionality. % ## Next version [elasticsearch-nextversion-deprecations] -% **Release date:** Month day, year -## 8.2.0 [elasticsearch-820-deprecations] -**Release date:** April 01, 2025 +```{applies_to} +stack: coming 9.1.0 +``` +## 9.1.0 [elasticsearch-9.1.0-deprecations] + +No deprecations in this version. + +```{applies_to} +stack: coming 9.0.10 +``` +## 9.0.10 [elasticsearch-9.0.10-deprecations] + +No deprecations in this version. + +## 9.0.9 [elasticsearch-9.0.9-deprecations] + +No deprecations in this version. + +## 9.0.8 [elasticsearch-9.0.8-deprecations] + +No deprecations in this version. + +## 9.0.7 [elasticsearch-9.0.7-deprecations] + +No deprecations in this version. + +## 9.0.6 [elasticsearch-9.0.6-deprecations] + +No deprecations in this version. + +## 9.0.5 [elasticsearch-9.0.5-deprecations] + +No deprecations in this version. + +## 9.0.4 [elasticsearch-9.0.4-deprecations] Search: -* Test changelog entry 4_0 [#4000](https://github.com/elastic/elasticsearch/pull/4000) (issue: {es-issue}4001[#4001]) -* Test changelog entry 4_1 [#4002](https://github.com/elastic/elasticsearch/pull/4002) (issues: {es-issue}4003[#4003], {es-issue}4004[#4004]) +* Test changelog entry 4_0 [#4000](https://github.com/elastic/elasticsearch/pull/4000) (issue: [#4001](https://github.com/elastic/elasticsearch/issues/4001)) +* Test changelog entry 4_1 [#4002](https://github.com/elastic/elasticsearch/pull/4002) (issues: [#4003](https://github.com/elastic/elasticsearch/issues/4003), [#4004](https://github.com/elastic/elasticsearch/issues/4004)) + + + +## 9.0.3 [elasticsearch-9.0.3-deprecations] + +No deprecations in this version. + +## 9.0.2 [elasticsearch-9.0.2-deprecations] + +No deprecations in this version. + +## 9.0.1 [elasticsearch-9.0.1-deprecations] + +No deprecations in this version. +## 9.0.0 [elasticsearch-900-deprecations] +No deprecations in this version. diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md index dc4320f684535..52c20341e1b5c 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md @@ -1,10 +1,7 @@ --- navigation_title: "Elasticsearch" mapped_pages: - - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-8.2.0.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.2.html --- # Elasticsearch release notes [elasticsearch-release-notes] @@ -16,7 +13,6 @@ To check for security updates, go to [Security announcements for the Elastic sta % Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. % ## version.next [elasticsearch-next-release-notes] -% **Release date:** Month day, year % ### Features and enhancements [elasticsearch-next-features-enhancements] % * @@ -24,68 +20,113 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * -## 8.2.0 [elasticsearch-820-release-notes] -**Release date:** April 01, 2025 +```{applies_to} +stack: coming 9.1.0 +``` +## 9.1.0 [elasticsearch-9.1.0-release-notes] -### Highlights [elasticsearch-820-highlights] +### Highlights [elasticsearch-9.1.0-highlights] -::::{dropdown} [Notable] Release highlight number 5000 -Release highlight body number 5000 - -For more information, check [PR #5000](https://github.com/elastic/elasticsearch/pull/5000). +::::{dropdown} [Notable] Release highlight number 50 +Release highlight body number 50 :::: -::::{dropdown} [Notable] Release highlight number 5002 -Release highlight body number 5002 +::::{dropdown} [Notable] Release highlight number 52 +Release highlight body number 52 +:::: -For more information, check [PR #5002](https://github.com/elastic/elasticsearch/pull/5002). +::::{dropdown} Release highlight number 51 +Release highlight body number 51 :::: -::::{dropdown} Release highlight number 5001 -Release highlight body number 5001 +### Features and enhancements [elasticsearch-9.1.0-features-enhancements] -For more information, check [PR #5001](https://github.com/elastic/elasticsearch/pull/5001). -:::: +Search: +* [#51](https://github.com/elastic/elasticsearch/pull/51) +* [#50](https://github.com/elastic/elasticsearch/pull/50) +* [#52](https://github.com/elastic/elasticsearch/pull/52) -### Features and enhancements [elasticsearch-820-features-enhancements] -Aggregation: -* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: [#6000](https://github.com/elastic/elasticsearch/pull/6000)) -* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: [#6002](https://github.com/elastic/elasticsearch/pull/6002), [#6002](https://github.com/elastic/elasticsearch/pull/6002)) +```{applies_to} +stack: coming 9.0.10 +``` +## 9.0.10 [elasticsearch-9.0.10-release-notes] -Cluster: -* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: [#7000](https://github.com/elastic/elasticsearch/pull/7000)) -* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: [#7002](https://github.com/elastic/elasticsearch/pull/7002), [#7002](https://github.com/elastic/elasticsearch/pull/7002)) +### Features and enhancements [elasticsearch-9.0.10-features-enhancements] + +Search: +* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: [#10001](https://github.com/elastic/elasticsearch/issues/10001)) +* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: [#10003](https://github.com/elastic/elasticsearch/issues/10003), [#10004](https://github.com/elastic/elasticsearch/issues/10004)) + + +## 9.0.9 [elasticsearch-9.0.9-release-notes] + +### Regressions [elasticsearch-9.0.9-regression] + +Mappings: +* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: [#9001](https://github.com/elastic/elasticsearch/issues/9001)) +* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: [#9003](https://github.com/elastic/elasticsearch/issues/9003), [#9004](https://github.com/elastic/elasticsearch/issues/9004)) + + +## 9.0.8 [elasticsearch-9.0.8-release-notes] + +### Features and enhancements [elasticsearch-9.0.8-features-enhancements] Indices: -* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: [#8000](https://github.com/elastic/elasticsearch/pull/8000)) -* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: [#8002](https://github.com/elastic/elasticsearch/pull/8002), [#8002](https://github.com/elastic/elasticsearch/pull/8002)) +* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: [#8001](https://github.com/elastic/elasticsearch/issues/8001)) +* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: [#8003](https://github.com/elastic/elasticsearch/issues/8003), [#8004](https://github.com/elastic/elasticsearch/issues/8004)) -Search: -* [#5002](https://github.com/elastic/elasticsearch/pull/5002) -* [#5000](https://github.com/elastic/elasticsearch/pull/5000) -* [#5001](https://github.com/elastic/elasticsearch/pull/5001) -* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: [#10000](https://github.com/elastic/elasticsearch/pull/10000)) -* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: [#10002](https://github.com/elastic/elasticsearch/pull/10002), [#10002](https://github.com/elastic/elasticsearch/pull/10002)) + +## 9.0.7 [elasticsearch-9.0.7-release-notes] + +### Features and enhancements [elasticsearch-9.0.7-features-enhancements] + +Cluster: +* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: [#7001](https://github.com/elastic/elasticsearch/issues/7001)) +* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: [#7003](https://github.com/elastic/elasticsearch/issues/7003), [#7004](https://github.com/elastic/elasticsearch/issues/7004)) + + +## 9.0.6 [elasticsearch-9.0.6-release-notes] + +### Features and enhancements [elasticsearch-9.0.6-features-enhancements] + +Aggregation: +* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: [#6001](https://github.com/elastic/elasticsearch/issues/6001)) +* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: [#6003](https://github.com/elastic/elasticsearch/issues/6003), [#6004](https://github.com/elastic/elasticsearch/issues/6004)) + + +## 9.0.5 [elasticsearch-9.0.5-release-notes] + +### Features and enhancements [elasticsearch-9.0.5-features-enhancements] Security: -* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: [#5000](https://github.com/elastic/elasticsearch/pull/5000)) -* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: [#5002](https://github.com/elastic/elasticsearch/pull/5002), [#5002](https://github.com/elastic/elasticsearch/pull/5002)) +* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: [#5001](https://github.com/elastic/elasticsearch/issues/5001)) +* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: [#5003](https://github.com/elastic/elasticsearch/issues/5003), [#5004](https://github.com/elastic/elasticsearch/issues/5004)) -### Fixes [elasticsearch-820-fixes] -Indices: -* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: [#2000](https://github.com/elastic/elasticsearch/pull/2000)) -* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: [#2002](https://github.com/elastic/elasticsearch/pull/2002), [#2002](https://github.com/elastic/elasticsearch/pull/2002)) +## 9.0.4 [elasticsearch-9.0.4-release-notes] -Mappings: -* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: [#3000](https://github.com/elastic/elasticsearch/pull/3000)) -* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: [#3002](https://github.com/elastic/elasticsearch/pull/3002), [#3002](https://github.com/elastic/elasticsearch/pull/3002)) -### Regressions [elasticsearch-820-regression] +## 9.0.3 [elasticsearch-9.0.3-release-notes] + +### Fixes [elasticsearch-9.0.3-fixes] Mappings: -* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: [#9000](https://github.com/elastic/elasticsearch/pull/9000)) -* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: [#9002](https://github.com/elastic/elasticsearch/pull/9002), [#9002](https://github.com/elastic/elasticsearch/pull/9002)) +* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: [#3001](https://github.com/elastic/elasticsearch/issues/3001)) +* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: [#3003](https://github.com/elastic/elasticsearch/issues/3003), [#3004](https://github.com/elastic/elasticsearch/issues/3004)) + + +## 9.0.2 [elasticsearch-9.0.2-release-notes] + +### Fixes [elasticsearch-9.0.2-fixes] + +Indices: +* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: [#2001](https://github.com/elastic/elasticsearch/issues/2001)) +* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: [#2003](https://github.com/elastic/elasticsearch/issues/2003), [#2004](https://github.com/elastic/elasticsearch/issues/2004)) + + +## 9.0.1 [elasticsearch-9.0.1-release-notes] + +## 9.0.0 [elasticsearch-900-release-notes] diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md index ea2b62653ae8b..5ec235cb3718a 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-changes.md @@ -1,10 +1,7 @@ --- navigation_title: "Elasticsearch" mapped_pages: - - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-8.2.0.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.2.html --- # Elasticsearch release notes [elasticsearch-release-notes] @@ -16,15 +13,9 @@ To check for security updates, go to [Security announcements for the Elastic sta % Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. % ## version.next [elasticsearch-next-release-notes] -% **Release date:** Month day, year % ### Features and enhancements [elasticsearch-next-features-enhancements] % * % ### Fixes [elasticsearch-next-fixes] % * - -## 8.2.0 [elasticsearch-820-release-notes] -**Release date:** April 01, 2025 - - diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md index 831c21251bbd1..ee81015b54f67 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md @@ -1,10 +1,7 @@ --- navigation_title: "Elasticsearch" mapped_pages: - - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-release-notes.html - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/release-notes-8.2.0.html - - https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.2.html --- # Elasticsearch release notes [elasticsearch-release-notes] @@ -16,7 +13,6 @@ To check for security updates, go to [Security announcements for the Elastic sta % Release notes include only features, enhancements, and fixes. Add breaking changes, deprecations, and known issues to the applicable release notes sections. % ## version.next [elasticsearch-next-release-notes] -% **Release date:** Month day, year % ### Features and enhancements [elasticsearch-next-features-enhancements] % * @@ -24,45 +20,86 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * -## 8.2.0 [elasticsearch-820-release-notes] -**Release date:** April 01, 2025 +```{applies_to} +stack: coming 9.0.10 +``` +## 9.0.10 [elasticsearch-9.0.10-release-notes] -### Features and enhancements [elasticsearch-820-features-enhancements] +### Features and enhancements [elasticsearch-9.0.10-features-enhancements] -Aggregation: -* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: [#6000](https://github.com/elastic/elasticsearch/pull/6000)) -* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: [#6002](https://github.com/elastic/elasticsearch/pull/6002), [#6002](https://github.com/elastic/elasticsearch/pull/6002)) +Search: +* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: [#10001](https://github.com/elastic/elasticsearch/issues/10001)) +* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: [#10003](https://github.com/elastic/elasticsearch/issues/10003), [#10004](https://github.com/elastic/elasticsearch/issues/10004)) -Cluster: -* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: [#7000](https://github.com/elastic/elasticsearch/pull/7000)) -* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: [#7002](https://github.com/elastic/elasticsearch/pull/7002), [#7002](https://github.com/elastic/elasticsearch/pull/7002)) + +## 9.0.9 [elasticsearch-9.0.9-release-notes] + +### Regressions [elasticsearch-9.0.9-regression] + +Mappings: +* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: [#9001](https://github.com/elastic/elasticsearch/issues/9001)) +* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: [#9003](https://github.com/elastic/elasticsearch/issues/9003), [#9004](https://github.com/elastic/elasticsearch/issues/9004)) + + +## 9.0.8 [elasticsearch-9.0.8-release-notes] + +### Features and enhancements [elasticsearch-9.0.8-features-enhancements] Indices: -* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: [#8000](https://github.com/elastic/elasticsearch/pull/8000)) -* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: [#8002](https://github.com/elastic/elasticsearch/pull/8002), [#8002](https://github.com/elastic/elasticsearch/pull/8002)) +* Test changelog entry 8_0 [#8000](https://github.com/elastic/elasticsearch/pull/8000) (issue: [#8001](https://github.com/elastic/elasticsearch/issues/8001)) +* Test changelog entry 8_1 [#8002](https://github.com/elastic/elasticsearch/pull/8002) (issues: [#8003](https://github.com/elastic/elasticsearch/issues/8003), [#8004](https://github.com/elastic/elasticsearch/issues/8004)) -Search: -* Test changelog entry 10_0 [#10000](https://github.com/elastic/elasticsearch/pull/10000) (issue: [#10000](https://github.com/elastic/elasticsearch/pull/10000)) -* Test changelog entry 10_1 [#10002](https://github.com/elastic/elasticsearch/pull/10002) (issues: [#10002](https://github.com/elastic/elasticsearch/pull/10002), [#10002](https://github.com/elastic/elasticsearch/pull/10002)) + +## 9.0.7 [elasticsearch-9.0.7-release-notes] + +### Features and enhancements [elasticsearch-9.0.7-features-enhancements] + +Cluster: +* Test changelog entry 7_0 [#7000](https://github.com/elastic/elasticsearch/pull/7000) (issue: [#7001](https://github.com/elastic/elasticsearch/issues/7001)) +* Test changelog entry 7_1 [#7002](https://github.com/elastic/elasticsearch/pull/7002) (issues: [#7003](https://github.com/elastic/elasticsearch/issues/7003), [#7004](https://github.com/elastic/elasticsearch/issues/7004)) + + +## 9.0.6 [elasticsearch-9.0.6-release-notes] + +### Features and enhancements [elasticsearch-9.0.6-features-enhancements] + +Aggregation: +* Test changelog entry 6_0 [#6000](https://github.com/elastic/elasticsearch/pull/6000) (issue: [#6001](https://github.com/elastic/elasticsearch/issues/6001)) +* Test changelog entry 6_1 [#6002](https://github.com/elastic/elasticsearch/pull/6002) (issues: [#6003](https://github.com/elastic/elasticsearch/issues/6003), [#6004](https://github.com/elastic/elasticsearch/issues/6004)) + + +## 9.0.5 [elasticsearch-9.0.5-release-notes] + +### Features and enhancements [elasticsearch-9.0.5-features-enhancements] Security: -* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: [#5000](https://github.com/elastic/elasticsearch/pull/5000)) -* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: [#5002](https://github.com/elastic/elasticsearch/pull/5002), [#5002](https://github.com/elastic/elasticsearch/pull/5002)) +* Test changelog entry 5_0 [#5000](https://github.com/elastic/elasticsearch/pull/5000) (issue: [#5001](https://github.com/elastic/elasticsearch/issues/5001)) +* Test changelog entry 5_1 [#5002](https://github.com/elastic/elasticsearch/pull/5002) (issues: [#5003](https://github.com/elastic/elasticsearch/issues/5003), [#5004](https://github.com/elastic/elasticsearch/issues/5004)) -### Fixes [elasticsearch-820-fixes] -Indices: -* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: [#2000](https://github.com/elastic/elasticsearch/pull/2000)) -* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: [#2002](https://github.com/elastic/elasticsearch/pull/2002), [#2002](https://github.com/elastic/elasticsearch/pull/2002)) +## 9.0.4 [elasticsearch-9.0.4-release-notes] -Mappings: -* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: [#3000](https://github.com/elastic/elasticsearch/pull/3000)) -* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: [#3002](https://github.com/elastic/elasticsearch/pull/3002), [#3002](https://github.com/elastic/elasticsearch/pull/3002)) -### Regressions [elasticsearch-820-regression] +## 9.0.3 [elasticsearch-9.0.3-release-notes] + +### Fixes [elasticsearch-9.0.3-fixes] Mappings: -* Test changelog entry 9_0 [#9000](https://github.com/elastic/elasticsearch/pull/9000) (issue: [#9000](https://github.com/elastic/elasticsearch/pull/9000)) -* Test changelog entry 9_1 [#9002](https://github.com/elastic/elasticsearch/pull/9002) (issues: [#9002](https://github.com/elastic/elasticsearch/pull/9002), [#9002](https://github.com/elastic/elasticsearch/pull/9002)) +* Test changelog entry 3_0 [#3000](https://github.com/elastic/elasticsearch/pull/3000) (issue: [#3001](https://github.com/elastic/elasticsearch/issues/3001)) +* Test changelog entry 3_1 [#3002](https://github.com/elastic/elasticsearch/pull/3002) (issues: [#3003](https://github.com/elastic/elasticsearch/issues/3003), [#3004](https://github.com/elastic/elasticsearch/issues/3004)) + + +## 9.0.2 [elasticsearch-9.0.2-release-notes] + +### Fixes [elasticsearch-9.0.2-fixes] + +Indices: +* Test changelog entry 2_0 [#2000](https://github.com/elastic/elasticsearch/pull/2000) (issue: [#2001](https://github.com/elastic/elasticsearch/issues/2001)) +* Test changelog entry 2_1 [#2002](https://github.com/elastic/elasticsearch/pull/2002) (issues: [#2003](https://github.com/elastic/elasticsearch/issues/2003), [#2004](https://github.com/elastic/elasticsearch/issues/2004)) + + +## 9.0.1 [elasticsearch-9.0.1-release-notes] + +## 9.0.0 [elasticsearch-900-release-notes] From 8efc0eac28e512595eae23da3ff8e5288ec22fac Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 19 May 2025 21:09:42 +0000 Subject: [PATCH 29/34] [CI] Auto commit changes from spotless --- .../gradle/internal/release/GenerateReleaseNotesTask.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index 62ebbdf99bd49..c3511cecd1d94 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -36,13 +36,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Stream; import javax.inject.Inject; -import static java.util.stream.Collectors.toSet; - /** * Orchestrates the steps required to generate or update various release notes files. */ From 09f1810b132c96bbab2022ceea50de282fc94bde Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 20 May 2025 11:13:07 -0400 Subject: [PATCH 30/34] Fixes --- .../gradle/internal/release/PruneChangelogsTask.java | 3 ++- .../gradle/internal/release/ReleaseNotesGeneratorTest.java | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java index 489b3561150fc..26236779a2390 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java @@ -149,7 +149,8 @@ static Stream findPreviousVersion(GitWrapper gitWrapper, Quali final String currentMajorPattern = "v" + version.major() + ".*"; final String previousMajorPattern = "v" + (version.major() - 1) + ".*"; - return gitWrapper.listVersions(previousMajorPattern).filter(v -> v.isBefore(version)); + return Stream.concat(gitWrapper.listVersions(currentMajorPattern), gitWrapper.listVersions(previousMajorPattern)) + .filter(v -> v.isBefore(version)); } /** diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java index b57b9feb137a0..248f59ad0fe18 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java @@ -13,7 +13,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -91,8 +90,6 @@ public void testTemplate(String templateFilename, String outputFilename, List Date: Thu, 22 May 2025 11:20:41 -0400 Subject: [PATCH 31/34] Generate 9.0.2 release notes --- .../release/BundleChangelogsTask.java | 4 +- docs/release-notes/breaking-changes.md | 10 ++ .../release-notes/changelog-bundles/9.0.2.yml | 135 ++++++++++++++++++ docs/release-notes/deprecations.md | 7 + docs/release-notes/index.md | 62 ++++++++ 5 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 docs/release-notes/changelog-bundles/9.0.2.yml diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index 935cf6e3bb8a3..27404fbae4708 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -132,10 +132,10 @@ public void executeTask() throws IOException { checkoutChangelogs(gitWrapper, upstreamRemote, branch); } Properties props = new Properties(); - props.load(new StringReader(gitWrapper.runCommand("git", "show", branch + ":build-tools-internal/version.properties"))); + props.load(new StringReader(gitWrapper.runCommand("git", "show", upstreamRemote + "/" + branch + ":build-tools-internal/version.properties"))); String version = props.getProperty("elasticsearch"); - LOGGER.info("Finding changelog files..."); + LOGGER.info("Finding changelog files for " + version + "..."); Set finalEntriesFromBc = entriesFromBc; List entries = this.changelogDirectory.getAsFileTree().getFiles().stream().filter(f -> { diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index 90ed5a0e67009..7309c76c5e450 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -12,6 +12,16 @@ If you are migrating from a version prior to version 9.0, you must first upgrade % ## Next version [elasticsearch-nextversion-breaking-changes] +```{applies_to} +stack: coming 9.0.2 +``` +## 9.0.2 [elasticsearch-9.0.2-breaking-changes] + +Snapshot/Restore: +* Make S3 custom query parameter optional [#128043](https://github.com/elastic/elasticsearch/pull/128043) + + + ## 9.0.1 [elasticsearch-9.0.1-breaking-changes] No breaking changes in this version. diff --git a/docs/release-notes/changelog-bundles/9.0.2.yml b/docs/release-notes/changelog-bundles/9.0.2.yml new file mode 100644 index 0000000000000..4b5a24f64bca0 --- /dev/null +++ b/docs/release-notes/changelog-bundles/9.0.2.yml @@ -0,0 +1,135 @@ +version: 9.0.2 +released: false +generated: 2025-05-22T15:14:00.768080Z +changelogs: + - pr: 126992 + summary: Add missing `outbound_network` entitlement to x-pack-core + area: Infra/Core + type: bug + issues: + - 127003 + - pr: 127009 + summary: "ESQL: Keep `DROP` attributes when resolving field names" + area: ES|QL + type: bug + issues: + - 126418 + - pr: 127337 + summary: Http proxy support in JWT realm + area: Authentication + type: enhancement + issues: + - 114956 + - pr: 127383 + summary: Don't push down filters on the right hand side of an inlinejoin + area: ES|QL + type: bug + issues: [] + - pr: 127475 + summary: Remove dangling spaces wherever found + area: Security + type: bug + issues: [] + - pr: 127563 + summary: "ESQL: Avoid unintended attribute removal" + area: ES|QL + type: bug + issues: + - 127468 + - pr: 127658 + summary: Append all data to Chat Completion buffer + area: Machine Learning + type: bug + issues: [] + - pr: 127687 + summary: "ESQL: Fix alias removal in regex extraction with JOIN" + area: ES|QL + type: bug + issues: + - 127467 + - pr: 127752 + summary: Downsampling does not consider passthrough fields as dimensions + area: Downsampling + type: bug + issues: + - 125156 + - pr: 127798 + summary: Handle streaming request body in audit log + area: Audit + type: bug + issues: [] + - pr: 127824 + summary: Skip the validation when retrieving the index mode during reindexing a time series data stream + area: TSDB + type: bug + issues: [] + - pr: 127856 + summary: Fix services API Google Vertex AI Rerank location field requirement + area: Machine Learning + type: bug + issues: [] + - pr: 127877 + summary: Check hidden frames in entitlements + area: Infra/Core + type: bug + issues: [] + - pr: 127921 + summary: "[9.x] Revert \"Enable madvise by default for all builds\"" + area: Vector Search + type: bug + issues: [] + - pr: 127924 + summary: Limit Replace function memory usage + area: ES|QL + type: enhancement + issues: [] + - pr: 127949 + summary: Ensure ordinal builder emit ordinal blocks + area: ES|QL + type: bug + issues: [] + - pr: 127975 + summary: Fix a bug in `significant_terms` + area: Aggregations + type: bug + issues: [] + - pr: 127991 + summary: Avoid nested docs in painless execute api + area: Infra/Scripting + type: bug + issues: + - 41004 + - pr: 128043 + summary: Make S3 custom query parameter optional + area: Snapshot/Restore + type: breaking + issues: [] + breaking: + area: Cluster and node setting + title: Make S3 custom query parameter optional + details: "Earlier versions of Elasticsearch would record the purpose of each S3 API call using the `?x-purpose=` custom query parameter. This isn't believed to be necessary outside of the ECH/ECE/ECK/... managed services, and it adds rather a lot to the request logs, so with this change we make the feature optional and disabled by default." + impact: "If you wish to reinstate the old behaviour on a S3 repository, set `s3.client.${CLIENT_NAME}.add_purpose_custom_query_parameter` to `true` for the relevant client." + notable: false + essSettingChange: false + - pr: 128047 + summary: Add missing entitlement to `repository-azure` + area: Snapshot/Restore + type: bug + issues: + - 128046 + - pr: 128111 + summary: Fix union types in CCS + area: ES|QL + type: bug + issues: [] + - pr: 128153 + summary: "Fix: Add `NamedWriteable` for `RuleQueryRankDoc`" + area: Relevance + type: bug + issues: + - 126071 + - pr: 128161 + summary: Fix system data streams incorrectly showing up in the list of template validation problems + area: Data streams + type: bug + issues: [] diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index a691d9e28a1e2..978066708c5c6 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -16,6 +16,13 @@ To give you insight into what deprecated features you’re using, {{es}}: % ## Next version [elasticsearch-nextversion-deprecations] +```{applies_to} +stack: coming 9.0.2 +``` +## 9.0.2 [elasticsearch-9.0.2-deprecations] + +No deprecations in this version. + ## 9.0.1 [elasticsearch-9.0.1-deprecations] No deprecations in this version. diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 4e841acf18ba3..eadc0f08b1d38 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -20,6 +20,68 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [elasticsearch-next-fixes] % * +```{applies_to} +stack: coming 9.0.2 +``` +## 9.0.2 [elasticsearch-9.0.2-release-notes] + +### Features and enhancements [elasticsearch-9.0.2-features-enhancements] + +Authentication: +* Http proxy support in JWT realm [#127337](https://github.com/elastic/elasticsearch/pull/127337) (issue: [#114956](https://github.com/elastic/elasticsearch/issues/114956)) + +ES|QL: +* Limit Replace function memory usage [#127924](https://github.com/elastic/elasticsearch/pull/127924) + +### Fixes [elasticsearch-9.0.2-fixes] + +Aggregations: +* Fix a bug in `significant_terms` [#127975](https://github.com/elastic/elasticsearch/pull/127975) + +Audit: +* Handle streaming request body in audit log [#127798](https://github.com/elastic/elasticsearch/pull/127798) + +Data streams: +* Fix system data streams incorrectly showing up in the list of template validation problems [#128161](https://github.com/elastic/elasticsearch/pull/128161) + +Downsampling: +* Downsampling does not consider passthrough fields as dimensions [#127752](https://github.com/elastic/elasticsearch/pull/127752) (issue: [#125156](https://github.com/elastic/elasticsearch/issues/125156)) + +ES|QL: +* Don't push down filters on the right hand side of an inlinejoin [#127383](https://github.com/elastic/elasticsearch/pull/127383) +* ESQL: Avoid unintended attribute removal [#127563](https://github.com/elastic/elasticsearch/pull/127563) (issue: [#127468](https://github.com/elastic/elasticsearch/issues/127468)) +* ESQL: Fix alias removal in regex extraction with JOIN [#127687](https://github.com/elastic/elasticsearch/pull/127687) (issue: [#127467](https://github.com/elastic/elasticsearch/issues/127467)) +* ESQL: Keep `DROP` attributes when resolving field names [#127009](https://github.com/elastic/elasticsearch/pull/127009) (issue: [#126418](https://github.com/elastic/elasticsearch/issues/126418)) +* Ensure ordinal builder emit ordinal blocks [#127949](https://github.com/elastic/elasticsearch/pull/127949) +* Fix union types in CCS [#128111](https://github.com/elastic/elasticsearch/pull/128111) + +Infra/Core: +* Add missing `outbound_network` entitlement to x-pack-core [#126992](https://github.com/elastic/elasticsearch/pull/126992) (issue: [#127003](https://github.com/elastic/elasticsearch/issues/127003)) +* Check hidden frames in entitlements [#127877](https://github.com/elastic/elasticsearch/pull/127877) + +Infra/Scripting: +* Avoid nested docs in painless execute api [#127991](https://github.com/elastic/elasticsearch/pull/127991) (issue: [#41004](https://github.com/elastic/elasticsearch/issues/41004)) + +Machine Learning: +* Append all data to Chat Completion buffer [#127658](https://github.com/elastic/elasticsearch/pull/127658) +* Fix services API Google Vertex AI Rerank location field requirement [#127856](https://github.com/elastic/elasticsearch/pull/127856) + +Relevance: +* Fix: Add `NamedWriteable` for `RuleQueryRankDoc` [#128153](https://github.com/elastic/elasticsearch/pull/128153) (issue: [#126071](https://github.com/elastic/elasticsearch/issues/126071)) + +Security: +* Remove dangling spaces wherever found [#127475](https://github.com/elastic/elasticsearch/pull/127475) + +Snapshot/Restore: +* Add missing entitlement to `repository-azure` [#128047](https://github.com/elastic/elasticsearch/pull/128047) (issue: [#128046](https://github.com/elastic/elasticsearch/issues/128046)) + +TSDB: +* Skip the validation when retrieving the index mode during reindexing a time series data stream [#127824](https://github.com/elastic/elasticsearch/pull/127824) + +Vector Search: +* [9.x] Revert "Enable madvise by default for all builds" [#127921](https://github.com/elastic/elasticsearch/pull/127921) + + ## 9.0.1 [elasticsearch-9.0.1-release-notes] ### Features and enhancements [elasticsearch-9.0.1-features-enhancements] From 27a612f0bf7e10b95f158a512d51afd6a5a0d296 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 22 May 2025 13:16:51 -0400 Subject: [PATCH 32/34] Cleanup/etc --- .../release/BundleChangelogsTask.java | 74 ++++++++++--------- .../release/GenerateReleaseNotesTask.java | 17 ----- .../internal/release/ReleaseToolsPlugin.java | 22 ++---- 3 files changed, 48 insertions(+), 65 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index 27404fbae4708..b87df14f45295 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -53,6 +53,7 @@ public class BundleChangelogsTask extends DefaultTask { private final RegularFileProperty bundleFile; private final DirectoryProperty changelogDirectory; + private final DirectoryProperty changelogBundlesDirectory; private final GitWrapper gitWrapper; @@ -60,16 +61,8 @@ public class BundleChangelogsTask extends DefaultTask { private String branch; @Nullable private String bcRef; - // private boolean updateExisting; - private boolean finalize; - // @Option( - // option = "update-existing", - // description = "Only update entries that are already in the bundle. Useful for updating the bundle after a BC has been cut." - // ) - // public void setUpdateExisting(boolean updateExisting) { - // this.updateExisting = updateExisting; - // } + private boolean finalize; @Option(option = "branch", description = "Branch (or other ref) to use for generating the changelog bundle.") public void setBranch(String branch) { @@ -105,6 +98,7 @@ public BundleChangelogsTask(ObjectFactory objectFactory, ExecOperations execOper bundleFile = objectFactory.fileProperty(); changelogDirectory = objectFactory.directoryProperty(); + changelogBundlesDirectory = objectFactory.directoryProperty(); gitWrapper = new GitWrapper(execOperations); } @@ -118,12 +112,13 @@ public void executeTask() throws IOException { final String upstreamRemote = gitWrapper.getUpstream(); Set entriesFromBc = Set.of(); + var didCheckoutChangelogs = false; try { var usingBcRef = bcRef != null && bcRef.isEmpty() == false; if (usingBcRef) { // Check out all the changelogs that existed at the time of the BC checkoutChangelogs(gitWrapper, upstreamRemote, bcRef); - entriesFromBc = this.changelogDirectory.getAsFileTree().getFiles().stream().map(File::getName).collect(Collectors.toSet()); + entriesFromBc = changelogDirectory.getAsFileTree().getFiles().stream().map(File::getName).collect(Collectors.toSet()); // Then add/update changelogs from the HEAD of the branch // We do an "add" here, rather than checking out the entire directory, in case changelogs have been removed for some reason @@ -131,14 +126,20 @@ public void executeTask() throws IOException { } else { checkoutChangelogs(gitWrapper, upstreamRemote, branch); } + + didCheckoutChangelogs = true; Properties props = new Properties(); - props.load(new StringReader(gitWrapper.runCommand("git", "show", upstreamRemote + "/" + branch + ":build-tools-internal/version.properties"))); + props.load( + new StringReader( + gitWrapper.runCommand("git", "show", upstreamRemote + "/" + branch + ":build-tools-internal/version.properties") + ) + ); String version = props.getProperty("elasticsearch"); LOGGER.info("Finding changelog files for " + version + "..."); Set finalEntriesFromBc = entriesFromBc; - List entries = this.changelogDirectory.getAsFileTree().getFiles().stream().filter(f -> { + List entries = changelogDirectory.getAsFileTree().getFiles().stream().filter(f -> { // When not using a bc ref, we just take everything from the branch/sha passed in if (usingBcRef == false) { return true; @@ -157,42 +158,38 @@ public void executeTask() throws IOException { return output.trim().isEmpty() == false; }).map(ChangelogEntry::parse).sorted(Comparator.comparing(ChangelogEntry::getPr)).collect(toList()); - ChangelogBundle existingBundle = null; - // if (updateExisting) { - // var existingBundleFile = new File("docs/release-notes/changelog-bundles/" + version + ".yml"); - // if (existingBundleFile.exists()) { - // var bundle = ChangelogBundle.parse(existingBundleFile); - // existingBundle = bundle; - // entries = entries.stream() - // .filter(e -> bundle.changelogs().stream().anyMatch(c -> c.getPr().equals(e.getPr()))) - // .toList(); - // } - // } - - var isReleased = finalize == true; - - ChangelogBundle bundle = new ChangelogBundle(version, isReleased, Instant.now().toString(), entries); + ChangelogBundle bundle = new ChangelogBundle(version, finalize, Instant.now().toString(), entries); yamlMapper.writeValue(new File("docs/release-notes/changelog-bundles/" + version + ".yml"), bundle); } finally { - gitWrapper.runCommand("git", "restore", "-s@", "-SW", "--", "docs/changelog"); + if (didCheckoutChangelogs) { + gitWrapper.runCommand("git", "restore", "-s@", "-SW", "--", changelogDirectory.get().toString()); + } } } - private static void checkoutChangelogs(GitWrapper gitWrapper, String upstream, String ref) { + private void checkoutChangelogs(GitWrapper gitWrapper, String upstream, String ref) { gitWrapper.updateRemote(upstream); - // TODO check for changes first - gitWrapper.runCommand("rm", "-rf", "docs/changelog"); + + // If the changelog directory contains modified/new files, we should error out instead of wiping them out silently + var output = gitWrapper.runCommand("git", "status", "--porcelain", changelogDirectory.get().toString()).trim(); + if (output.isEmpty() == false) { + throw new IllegalStateException( + "Changelog directory contains changes that will be wiped out by this task:\n" + changelogDirectory.get() + "\n" + output + ); + } + + gitWrapper.runCommand("rm", "-rf", changelogDirectory.get().toString()); var refSpec = upstream + "/" + ref; if (ref.contains("upstream/")) { refSpec = ref.replace("upstream/", upstream + "/"); } else if (ref.matches("^[0-9a-f]+$")) { refSpec = ref; } - gitWrapper.runCommand("git", "checkout", refSpec, "--", "docs/changelog"); + gitWrapper.runCommand("git", "checkout", refSpec, "--", changelogDirectory.get().toString()); } - private static void addChangelogsFromRef(GitWrapper gitWrapper, String upstream, String ref) { + private void addChangelogsFromRef(GitWrapper gitWrapper, String upstream, String ref) { var refSpec = upstream + "/" + ref; if (ref.contains("upstream/")) { refSpec = ref.replace("upstream/", upstream + "/"); @@ -200,7 +197,7 @@ private static void addChangelogsFromRef(GitWrapper gitWrapper, String upstream, refSpec = ref; } - gitWrapper.runCommand("git", "checkout", refSpec, "--", "docs/changelog/*.yaml"); + gitWrapper.runCommand("git", "checkout", refSpec, "--", changelogDirectory.get() + "/*.yaml"); } @InputDirectory @@ -212,6 +209,15 @@ public void setChangelogDirectory(Directory dir) { this.changelogDirectory.set(dir); } + @InputDirectory + public DirectoryProperty getChangelogBundlesDirectory() { + return changelogBundlesDirectory; + } + + public void setChangelogBundlesDirectory(Directory dir) { + this.changelogBundlesDirectory.set(dir); + } + @InputFiles public FileCollection getChangelogs() { return changelogs; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index c3511cecd1d94..921d980d34368 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -14,10 +14,8 @@ import org.elasticsearch.gradle.VersionProperties; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; -import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.Directory; import org.gradle.api.file.DirectoryProperty; -import org.gradle.api.file.FileCollection; import org.gradle.api.file.RegularFile; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; @@ -25,7 +23,6 @@ import org.gradle.api.model.ObjectFactory; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFile; -import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; import org.gradle.process.ExecOperations; @@ -45,8 +42,6 @@ public class GenerateReleaseNotesTask extends DefaultTask { private static final Logger LOGGER = Logging.getLogger(GenerateReleaseNotesTask.class); - private final ConfigurableFileCollection changelogs; - private final RegularFileProperty releaseNotesTemplate; private final RegularFileProperty releaseHighlightsTemplate; private final RegularFileProperty breakingChangesTemplate; @@ -63,8 +58,6 @@ public class GenerateReleaseNotesTask extends DefaultTask { @Inject public GenerateReleaseNotesTask(ObjectFactory objectFactory, ExecOperations execOperations) { - changelogs = objectFactory.fileCollection(); - releaseNotesTemplate = objectFactory.fileProperty(); releaseHighlightsTemplate = objectFactory.fileProperty(); breakingChangesTemplate = objectFactory.fileProperty(); @@ -107,7 +100,6 @@ public void executeTask() throws IOException { @VisibleForTesting static List getSortedBundlesWithUniqueChangelogs(List bundles) { List sorted = bundles.stream() - // .map(ChangelogBundle::copy) .sorted(Comparator.comparing(ChangelogBundle::released).reversed().thenComparing(ChangelogBundle::generated)) .toList(); @@ -176,15 +168,6 @@ static boolean needsGitTags(String versionString) { return true; } - @InputFiles - public FileCollection getChangelogs() { - return changelogs; - } - - public void setChangelogs(FileCollection files) { - this.changelogs.setFrom(files); - } - @InputDirectory public DirectoryProperty getChangelogBundleDirectory() { return changelogBundleDirectory; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index a696d034be0bf..ff1f4bd0f0cb3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -56,6 +56,7 @@ public void apply(Project project) { project.getTasks().register("setCompatibleVersions", SetCompatibleVersionsTask.class, t -> t.setThisVersion(version)); final Directory changeLogDirectory = projectDirectory.dir("docs/changelog"); + final Directory changeLogBundlesDirectory = projectDirectory.dir("docs/release-notes/changelog-bundles"); final FileTree yamlFiles = changeLogDirectory.getAsFileTree().matching(new PatternSet().include("**/*.yml", "**/*.yaml")); final Provider validateChangelogsTask = project.getTasks() @@ -67,23 +68,19 @@ public void apply(Project project) { task.setReport(new File(project.getBuildDir(), "reports/validateYaml.txt")); }); - final Function> configureBundleTask = shouldConfigureYamlFiles -> task -> { + final Action configureBundleTask = task -> { task.setGroup("Documentation"); task.setDescription("Generates release notes from changelog files held in this checkout"); task.setChangelogs(yamlFiles); task.setChangelogDirectory(changeLogDirectory); + task.setChangelogBundlesDirectory(changeLogBundlesDirectory); task.setBundleFile(projectDirectory.file("docs/release-notes/changelogs-" + version.toString() + ".yml")); task.getOutputs().upToDateWhen(o -> false); }; - final Function> configureGenerateTask = shouldConfigureYamlFiles -> task -> { + final Action configureGenerateTask = task -> { task.setGroup("Documentation"); - if (shouldConfigureYamlFiles) { - task.setChangelogs(yamlFiles); - task.setDescription("Generates release notes from changelog files held in this checkout"); - } else { - task.setDescription("Generates stub release notes e.g. after feature freeze"); - } + task.setDescription("Generates release notes for all versions/branches using changelog bundles in this checkout"); task.setReleaseNotesTemplate(projectDirectory.file(RESOURCES + "templates/index.md")); task.setReleaseNotesFile(projectDirectory.file("docs/release-notes/index.md")); @@ -97,18 +94,15 @@ public void apply(Project project) { task.setDeprecationsTemplate(projectDirectory.file(RESOURCES + "templates/deprecations.md")); task.setDeprecationsFile(projectDirectory.file("docs/release-notes/deprecations.md")); - task.setChangelogBundleDirectory(projectDirectory.dir("docs/release-notes/changelog-bundles")); + task.setChangelogBundleDirectory(changeLogBundlesDirectory); task.getOutputs().upToDateWhen(o -> false); task.dependsOn(validateChangelogsTask); }; - project.getTasks().register("bundleChangelogs", BundleChangelogsTask.class).configure(configureBundleTask.apply(true)); - project.getTasks().register("generateReleaseNotes", GenerateReleaseNotesTask.class).configure(configureGenerateTask.apply(true)); - project.getTasks() - .register("generateStubReleaseNotes", GenerateReleaseNotesTask.class) - .configure(configureGenerateTask.apply(false)); + project.getTasks().register("bundleChangelogs", BundleChangelogsTask.class).configure(configureBundleTask); + project.getTasks().register("generateReleaseNotes", GenerateReleaseNotesTask.class).configure(configureGenerateTask); project.getTasks().register("pruneChangelogs", PruneChangelogsTask.class).configure(task -> { task.setGroup("Documentation"); From d422d09dc56dded9b23ecbcfc9ed312cd7e10e5a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 22 May 2025 17:26:25 +0000 Subject: [PATCH 33/34] [CI] Auto commit changes from spotless --- .../gradle/internal/release/ReleaseToolsPlugin.java | 1 - 1 file changed, 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index ff1f4bd0f0cb3..fce8b0c545dbb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -23,7 +23,6 @@ import org.gradle.api.tasks.util.PatternSet; import java.io.File; -import java.util.function.Function; import javax.inject.Inject; From 8ff6ba19a6f47bc9c25d47589d4187ab81b9d09d Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 22 May 2025 16:03:10 -0400 Subject: [PATCH 34/34] Cleanup and comments --- .../internal/release/BundleChangelogsTask.java | 16 +++++++++++++++- .../release/GenerateReleaseNotesTask.java | 16 +--------------- .../internal/release/ReleaseNotesGenerator.java | 17 ----------------- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java index b87df14f45295..7749079a698d5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java @@ -103,6 +103,20 @@ public BundleChangelogsTask(ObjectFactory objectFactory, ExecOperations execOper gitWrapper = new GitWrapper(execOperations); } + /* + Given a branch, and possibly a build candidate commit sha + Check out the changelog yaml files from the branch/BC sha + Then, bundle them all up into one file and write it to disk, along with a timestamp and whether the release is considered released + + When using a branch without a BC sha: + - Check out the changelog yaml files from the HEAD of the branch + + When using a BC sha: + - Check out the changelog yaml files from the BC commit + - Update those files with any updates from the HEAD of the branch (in case the changelogs get modified later) + - Check for any changelog yaml files that were added AFTER the BC, + but whose PR was merged before the BC (in case someone adds a forgotten changelog after the fact) + */ @TaskAction public void executeTask() throws IOException { if (branch == null) { @@ -145,7 +159,7 @@ public void executeTask() throws IOException { return true; } - // If the changelog was present in the BC sha, use it + // If the changelog was present in the BC sha, always use it if (finalEntriesFromBc.contains(f.getName())) { return true; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index 921d980d34368..655af7ae98c12 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -13,7 +13,6 @@ import org.elasticsearch.gradle.VersionProperties; import org.gradle.api.DefaultTask; -import org.gradle.api.GradleException; import org.gradle.api.file.Directory; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.RegularFile; @@ -32,7 +31,6 @@ import java.util.Comparator; import java.util.HashSet; import java.util.List; -import java.util.Map; import javax.inject.Inject; @@ -82,7 +80,6 @@ public void executeTask() throws IOException { } LOGGER.info("Finding changelog bundles..."); - List allBundles = this.changelogBundleDirectory.getAsFileTree() .getFiles() .stream() @@ -126,18 +123,7 @@ static List getSortedBundlesWithUniqueChangelogs(List entry.getValue().contains("elastic/elasticsearch")) - .findFirst() - .map(Map.Entry::getKey) - .orElseThrow( - () -> new GradleException( - "I need to ensure the git tags are up-to-date, but I couldn't find a git remote for [elastic/elasticsearch]" - ) - ); + String upstream = gitWrapper.getUpstream(); LOGGER.info("Updating remote [{}]", upstream); // Now update the remote, and make sure we update the tags too diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java index 5979c654075c9..6e8b9d9fa8023 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java @@ -109,25 +109,8 @@ static String generateFile(String template, List bundles) throw bundlesWrapped.add(wrapped); } - // final var changelogsByTypeByArea = buildChangelogBreakdown(changelogs); - // - // final Map> groupedHighlights = changelogs.stream() - // .map(ChangelogEntry::getHighlight) - // .filter(Objects::nonNull) - // .sorted(comparingInt(ChangelogEntry.Highlight::getPr)) - // .collect(groupingBy(ChangelogEntry.Highlight::isNotable, toList())); - // - // final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); - // final List nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); - final Map bindings = new HashMap<>(); - // bindings.put("version", version); - // bindings.put("changelogsByTypeByArea", changelogsByTypeByArea); bindings.put("TYPE_LABELS", TYPE_LABELS); - // bindings.put("unqualifiedVersion", version.withoutQualifier()); - // bindings.put("versionWithoutSeparator", version.withoutQualifier().toString().replaceAll("\\.", "")); - // bindings.put("notableHighlights", notableHighlights); - // bindings.put("nonNotableHighlights", nonNotableHighlights); bindings.put("changelogBundles", bundlesWrapped); return TemplateUtils.render(template, bindings);