Merge branch 'main' into 2025/09/09/cluster-applier-thread-watchdog

Author: DaveCTurner

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java

@@ -22,9 +22,11 @@ public static TransportVersionDefinition fromString(Path file, String contents,
 
         String idsLine = null;
         if (contents.isEmpty() == false) {
-            String[] lines = contents.split(System.lineSeparator());
+            // Regardless of whether windows newlines exist (they could be added by git), we split on line feed.
+            // All we care about skipping lines with the comment character, so the remaining \r won't matter
+            String[] lines = contents.split("\n");
             for (String line : lines) {
-                line = line.replaceAll("\\s+", "");
+                line = line.strip();
                 if (line.startsWith("#") == false) {
                     idsLine = line;
                     break;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java

@@ -24,9 +24,11 @@ public static TransportVersionUpperBound fromString(Path file, String contents)
         String branch = filename.substring(slashIndex == -1 ? 0 : (slashIndex + 1), filename.length() - 4);
 
         String idsLine = null;
-        String[] lines = contents.split(System.lineSeparator());
+        // Regardless of whether windows newlines exist (they could be added by git), we split on line feed.
+        // All we care about skipping lines with the comment character, so the remaining \r won't matter
+        String[] lines = contents.split("\n");
         for (String line : lines) {
-            line = line.replaceAll("\\s+", "");
+            line = line.strip();
             if (line.startsWith("#") == false) {
                 idsLine = line;
                 break;

docs/changelog/135897.yaml

@@ -0,0 +1,5 @@
+pr: 135897
+summary: Apply source excludes early when retrieving the `_inference_fields`
+area: Search
+type: bug
+issues: []

docs/changelog/135901.yaml

@@ -0,0 +1,5 @@
+pr: 135901
+summary: Add support for dot-separated attribute names (e.g. `foo.bar`) and for parameters (e.g. `??my_param`) in FUSE GROUP BY
+area: ES|QL
+type: bug
+issues: []

docs/reference/elasticsearch/index-lifecycle-actions/ilm-downsample.md

@@ -7,7 +7,7 @@ mapped_pages:
 
 Phases allowed: hot, warm, cold.
 
-Aggregates a time series (TSDS) index and stores pre-computed statistical summaries (`min`, `max`, `sum`, `value_count` and `avg`) for each metric field grouped by a configured time interval. For example, a TSDS index that contains metrics sampled every 10 seconds can be downsampled to an hourly index. All documents within an hour interval are summarized and stored as a single document and stored in the downsample index.
+Aggregates a time series (TSDS) index and stores pre-computed statistical summaries (`min`, `max`, `sum`, and `value_count`) for each metric field grouped by a configured time interval. For example, a TSDS index that contains metrics sampled every 10 seconds can be downsampled to an hourly index. All documents within an hour interval are summarized and stored as a single document and stored in the downsample index.
 
 This action corresponds to the  [downsample API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-downsample).
 

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java

@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
 
@@ -38,10 +39,12 @@ public class RestSearchTemplateAction extends BaseRestHandler {
 
     private final Predicate<NodeFeature> clusterSupportsFeature;
     private final Settings settings;
+    private final boolean inCpsContext;
 
     public RestSearchTemplateAction(Predicate<NodeFeature> clusterSupportsFeature, Settings settings) {
         this.clusterSupportsFeature = clusterSupportsFeature;
         this.settings = settings;
+        this.inCpsContext = settings != null && settings.getAsBoolean("serverless.cross_project.enabled", false);
     }
 
     @Override
@@ -61,7 +64,7 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        if (settings != null && settings.getAsBoolean("serverless.cross_project.enabled", false)) {
+        if (inCpsContext) {
             // accept but drop project_routing param until fully supported
             request.param("project_routing");
         }
@@ -73,7 +76,9 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
             request,
             null,
             clusterSupportsFeature,
-            size -> searchRequest.source().size(size)
+            size -> searchRequest.source().size(size),
+            // This endpoint is CPS-enabled so propagate the right value.
+            Optional.of(inCpsContext)
         );
 
         // Creates the search template request

modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java

@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
@@ -52,7 +53,14 @@ protected void parseInternalRequest(
         SearchRequest searchRequest = internal.getSearchRequest();
 
         try (XContentParser parser = extractRequestSpecificFields(restRequest, bodyConsumers)) {
-            RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, clusterSupportsFeature, size -> failOnSizeSpecified());
+            RestSearchAction.parseSearchRequest(
+                searchRequest,
+                restRequest,
+                parser,
+                clusterSupportsFeature,
+                size -> failOnSizeSpecified(),
+                Optional.empty()
+            );
         }
 
         searchRequest.source().size(restRequest.paramAsInt("scroll_size", searchRequest.source().size()));

muted-tests.yml

@@ -504,9 +504,6 @@ tests:
 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT
   method: testRepositoryAnalysis
   issue: https://github.com/elastic/elasticsearch/issues/134853
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
-  method: testStopQueryInlinestats
-  issue: https://github.com/elastic/elasticsearch/issues/134854
 - class: org.elasticsearch.xpack.esql.action.CrossClusterCancellationIT
   method: testCancelSkipUnavailable
   issue: https://github.com/elastic/elasticsearch/issues/134865
@@ -522,9 +519,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
   method: test {csv-spec:stats.CountDistinctWithConditions}
   issue: https://github.com/elastic/elasticsearch/issues/134993
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
-  method: testStopQueryInlineStats
-  issue: https://github.com/elastic/elasticsearch/issues/135032
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {csv-spec:fork.ForkBeforeStatsWithWhere}
   issue: https://github.com/elastic/elasticsearch/issues/135041
@@ -660,6 +654,75 @@ tests:
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
   method: testPersianAnalyzerBWC {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/135930
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSnapshotRestore {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135933
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testForbidDisableSoftDeletesOnRestore {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135937
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testShrinkAfterUpgrade {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135938
+- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
+  method: testTopNPushedToLuceneOnSortedIndex
+  issue: https://github.com/elastic/elasticsearch/issues/135939
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testClosedIndices {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135941
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testOperationBasedRecovery {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135944
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testRomanianAnalyzerBWC {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135948
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testEmptyShard {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135951
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testRollover {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135952
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testRecovery {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135953
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testNewReplicas {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135956
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testHistoryUUIDIsAdded {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135957
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testTurnOffTranslogRetentionAfterUpgraded {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135958
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSoftDeletes {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135959
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testClusterState {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135960
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testShrink {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135962
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSearchTimeSeriesMode {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135963
+- class: org.elasticsearch.datastreams.DataStreamIndexSettingsProviderTests
+  method: testGetAdditionalIndexSettings
+  issue: https://github.com/elastic/elasticsearch/issues/135972
+- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
+  method: test {yaml=indices.get_sample/10_basic/Test get sample for index with no sample config}
+  issue: https://github.com/elastic/elasticsearch/issues/135975
+- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
+  method: testStopQueryInlineStats
+  issue: https://github.com/elastic/elasticsearch/issues/135032
+- class: org.elasticsearch.xpack.esql.plan.logical.local.ImmediateLocalSupplierTests
+  method: testEqualsAndHashcode
+  issue: https://github.com/elastic/elasticsearch/issues/135977
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=indices.get_sample/10_basic/Test get sample for index with no sample config}
+  issue: https://github.com/elastic/elasticsearch/issues/135989
+- class: org.elasticsearch.compute.data.BasicPageTests
+  method: testEqualityAndHashCode
+  issue: https://github.com/elastic/elasticsearch/issues/135990
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -53,7 +53,6 @@ static TransportVersion def(int id) {
     }
 
     // TODO: ES-10337 we can remove all transport versions earlier than 8.18
-    public static final TransportVersion V_7_0_0 = def(7_00_00_99);
     public static final TransportVersion V_7_1_0 = def(7_01_00_99);
     public static final TransportVersion V_7_2_0 = def(7_02_00_99);
     public static final TransportVersion V_7_3_0 = def(7_03_00_99);

server/src/main/java/org/elasticsearch/index/get/ShardGetService.java

@@ -426,6 +426,14 @@ private static Boolean shouldExcludeVectorsFromSourceExplicit(FetchSourceContext
 
     public static boolean shouldExcludeInferenceFieldsFromSource(IndexSettings indexSettings, FetchSourceContext fetchSourceContext) {
         var explicit = shouldExcludeInferenceFieldsFromSourceExplicit(fetchSourceContext);
+        var filter = fetchSourceContext != null ? fetchSourceContext.filter() : null;
+        if (filter != null) {
+            if (filter.isPathFiltered(InferenceMetadataFieldsMapper.NAME, true)) {
+                return true;
+            } else if (filter.isExplicitlyIncluded(InferenceMetadataFieldsMapper.NAME)) {
+                return false;
+            }
+        }
         return explicit != null ? explicit : INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.get(indexSettings.getSettings());
     }