Merge branch 'main' into subquery-in-from

Author: fang-xing-esql

docs/changelog/135897.yaml

@@ -0,0 +1,5 @@
+pr: 135897
+summary: Apply source excludes early when retrieving the `_inference_fields`
+area: Search
+type: bug
+issues: []

docs/changelog/135901.yaml

@@ -0,0 +1,5 @@
+pr: 135901
+summary: Add support for dot-separated attribute names (e.g. `foo.bar`) and for parameters (e.g. `??my_param`) in FUSE GROUP BY
+area: ES|QL
+type: bug
+issues: []

docs/reference/elasticsearch/index-lifecycle-actions/ilm-downsample.md

@@ -7,7 +7,7 @@ mapped_pages:
 
 Phases allowed: hot, warm, cold.
 
-Aggregates a time series (TSDS) index and stores pre-computed statistical summaries (`min`, `max`, `sum`, `value_count` and `avg`) for each metric field grouped by a configured time interval. For example, a TSDS index that contains metrics sampled every 10 seconds can be downsampled to an hourly index. All documents within an hour interval are summarized and stored as a single document and stored in the downsample index.
+Aggregates a time series (TSDS) index and stores pre-computed statistical summaries (`min`, `max`, `sum`, and `value_count`) for each metric field grouped by a configured time interval. For example, a TSDS index that contains metrics sampled every 10 seconds can be downsampled to an hourly index. All documents within an hour interval are summarized and stored as a single document and stored in the downsample index.
 
 This action corresponds to the  [downsample API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-downsample).
 

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java

@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
 
@@ -38,10 +39,12 @@ public class RestSearchTemplateAction extends BaseRestHandler {
 
     private final Predicate<NodeFeature> clusterSupportsFeature;
     private final Settings settings;
+    private final boolean inCpsContext;
 
     public RestSearchTemplateAction(Predicate<NodeFeature> clusterSupportsFeature, Settings settings) {
         this.clusterSupportsFeature = clusterSupportsFeature;
         this.settings = settings;
+        this.inCpsContext = settings != null && settings.getAsBoolean("serverless.cross_project.enabled", false);
     }
 
     @Override
@@ -61,7 +64,7 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        if (settings != null && settings.getAsBoolean("serverless.cross_project.enabled", false)) {
+        if (inCpsContext) {
             // accept but drop project_routing param until fully supported
             request.param("project_routing");
         }
@@ -73,7 +76,9 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
             request,
             null,
             clusterSupportsFeature,
-            size -> searchRequest.source().size(size)
+            size -> searchRequest.source().size(size),
+            // This endpoint is CPS-enabled so propagate the right value.
+            Optional.of(inCpsContext)
         );
 
         // Creates the search template request

modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java

@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
@@ -52,7 +53,14 @@ protected void parseInternalRequest(
         SearchRequest searchRequest = internal.getSearchRequest();
 
         try (XContentParser parser = extractRequestSpecificFields(restRequest, bodyConsumers)) {
-            RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, clusterSupportsFeature, size -> failOnSizeSpecified());
+            RestSearchAction.parseSearchRequest(
+                searchRequest,
+                restRequest,
+                parser,
+                clusterSupportsFeature,
+                size -> failOnSizeSpecified(),
+                Optional.empty()
+            );
         }
 
         searchRequest.source().size(restRequest.paramAsInt("scroll_size", searchRequest.source().size()));

muted-tests.yml

@@ -504,9 +504,6 @@ tests:
 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT
   method: testRepositoryAnalysis
   issue: https://github.com/elastic/elasticsearch/issues/134853
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
-  method: testStopQueryInlinestats
-  issue: https://github.com/elastic/elasticsearch/issues/134854
 - class: org.elasticsearch.xpack.esql.action.CrossClusterCancellationIT
   method: testCancelSkipUnavailable
   issue: https://github.com/elastic/elasticsearch/issues/134865
@@ -522,9 +519,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
   method: test {csv-spec:stats.CountDistinctWithConditions}
   issue: https://github.com/elastic/elasticsearch/issues/134993
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
-  method: testStopQueryInlineStats
-  issue: https://github.com/elastic/elasticsearch/issues/135032
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {csv-spec:fork.ForkBeforeStatsWithWhere}
   issue: https://github.com/elastic/elasticsearch/issues/135041
@@ -648,6 +642,66 @@ tests:
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
   method: testSystemIndexMetadataIsUpgraded {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/135923
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSearch {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135927
+- class: org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilderTests
+  method: testBuildChunkingSettingsForElasticReranker_QueryTokenCountLessThanHalfOfTokenLimit
+  issue: https://github.com/elastic/elasticsearch/issues/135928
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testPeerRecoveryRetentionLeases {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135929
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testPersianAnalyzerBWC {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135930
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSnapshotRestore {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135933
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testForbidDisableSoftDeletesOnRestore {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135937
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testShrinkAfterUpgrade {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135938
+- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
+  method: testTopNPushedToLuceneOnSortedIndex
+  issue: https://github.com/elastic/elasticsearch/issues/135939
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testClosedIndices {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135941
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testOperationBasedRecovery {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135944
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testRomanianAnalyzerBWC {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135948
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testEmptyShard {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135951
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testRollover {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135952
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testRecovery {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135953
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testNewReplicas {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135956
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testHistoryUUIDIsAdded {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135957
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testTurnOffTranslogRetentionAfterUpgraded {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135958
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSoftDeletes {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135959
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testClusterState {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135960
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testShrink {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135962
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -53,7 +53,6 @@ static TransportVersion def(int id) {
     }
 
     // TODO: ES-10337 we can remove all transport versions earlier than 8.18
-    public static final TransportVersion V_7_0_0 = def(7_00_00_99);
     public static final TransportVersion V_7_1_0 = def(7_01_00_99);
     public static final TransportVersion V_7_2_0 = def(7_02_00_99);
     public static final TransportVersion V_7_3_0 = def(7_03_00_99);

server/src/main/java/org/elasticsearch/index/get/ShardGetService.java

@@ -426,6 +426,14 @@ private static Boolean shouldExcludeVectorsFromSourceExplicit(FetchSourceContext
 
     public static boolean shouldExcludeInferenceFieldsFromSource(IndexSettings indexSettings, FetchSourceContext fetchSourceContext) {
         var explicit = shouldExcludeInferenceFieldsFromSourceExplicit(fetchSourceContext);
+        var filter = fetchSourceContext != null ? fetchSourceContext.filter() : null;
+        if (filter != null) {
+            if (filter.isPathFiltered(InferenceMetadataFieldsMapper.NAME, true)) {
+                return true;
+            } else if (filter.isExplicitlyIncluded(InferenceMetadataFieldsMapper.NAME)) {
+                return false;
+            }
+        }
         return explicit != null ? explicit : INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.get(indexSettings.getSettings());
     }
 

server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java

@@ -43,6 +43,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.IntConsumer;
 import java.util.function.Predicate;
@@ -68,6 +69,7 @@ public class RestSearchAction extends BaseRestHandler {
     private final SearchUsageHolder searchUsageHolder;
     private final Predicate<NodeFeature> clusterSupportsFeature;
     private final Settings settings;
+    private final boolean inCpsContext;
 
     public RestSearchAction(SearchUsageHolder searchUsageHolder, Predicate<NodeFeature> clusterSupportsFeature) {
         this(searchUsageHolder, clusterSupportsFeature, null);
@@ -77,6 +79,7 @@ public RestSearchAction(SearchUsageHolder searchUsageHolder, Predicate<NodeFeatu
         this.searchUsageHolder = searchUsageHolder;
         this.clusterSupportsFeature = clusterSupportsFeature;
         this.settings = settings;
+        this.inCpsContext = settings != null && settings.getAsBoolean("serverless.cross_project.enabled", false);
     }
 
     @Override
@@ -109,7 +112,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         // this might be set by old clients
         request.param("min_compatible_shard_node");
 
-        if (settings != null && settings.getAsBoolean("serverless.cross_project.enabled", false)) {
+        if (inCpsContext) {
             // accept but drop project_routing param until fully supported
             request.param("project_routing");
         }
@@ -128,7 +131,16 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
          */
         IntConsumer setSize = size -> searchRequest.source().size(size);
         request.withContentOrSourceParamParserOrNull(
-            parser -> parseSearchRequest(searchRequest, request, parser, clusterSupportsFeature, setSize, searchUsageHolder)
+            parser -> parseSearchRequest(
+                searchRequest,
+                request,
+                parser,
+                clusterSupportsFeature,
+                setSize,
+                searchUsageHolder,
+                // This endpoint is CPS-enabled so propagate the right value.
+                Optional.of(inCpsContext)
+            )
         );
 
         return channel -> {
@@ -146,15 +158,23 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
      *        parameter
      * @param clusterSupportsFeature used to check if certain features are available in this cluster
      * @param setSize how the size url parameter is handled. {@code udpate_by_query} and regular search differ here.
+     * @param inCpsContext specifies if we're in CPS context.
+     *                     <br>
+     *                     true - the endpoint that's invoking this method is CPS-enabled and in a CPS/Serverless context.
+     *                     <br>
+     *                     false - the endpoint that's invoking this method is CPS-enabled but not in a CPS/Serverless context.
+     *                     <br>
+     *                     Optional.empty - the endpoint is not CPS-enabled irrespective of the environment.
      */
     public static void parseSearchRequest(
         SearchRequest searchRequest,
         RestRequest request,
         XContentParser requestContentParser,
         Predicate<NodeFeature> clusterSupportsFeature,
-        IntConsumer setSize
+        IntConsumer setSize,
+        Optional<Boolean> inCpsContext
     ) throws IOException {
-        parseSearchRequest(searchRequest, request, requestContentParser, clusterSupportsFeature, setSize, null);
+        parseSearchRequest(searchRequest, request, requestContentParser, clusterSupportsFeature, setSize, null, inCpsContext);
     }
 
     /**
@@ -167,14 +187,22 @@ public static void parseSearchRequest(
      * @param clusterSupportsFeature used to check if certain features are available in this cluster
      * @param setSize how the size url parameter is handled. {@code udpate_by_query} and regular search differ here.
      * @param searchUsageHolder the holder of search usage stats
+     * @param inCpsContext specifies if we're in CPS context.
+     *                     <br>
+     *                     true - the endpoint that's invoking this method is CPS-enabled and in a CPS/Serverless context.
+     *                     <br>
+     *                     false - the endpoint that's invoking this method is CPS-enabled but not in a CPS/Serverless context.
+     *                     <br>
+     *                     Optional.empty - the endpoint is not CPS-enabled irrespective of the environment.
      */
     public static void parseSearchRequest(
         SearchRequest searchRequest,
         RestRequest request,
         @Nullable XContentParser requestContentParser,
         Predicate<NodeFeature> clusterSupportsFeature,
         IntConsumer setSize,
-        @Nullable SearchUsageHolder searchUsageHolder
+        @Nullable SearchUsageHolder searchUsageHolder,
+        Optional<Boolean> inCpsContext
     ) throws IOException {
         if (searchRequest.source() == null) {
             searchRequest.source(new SearchSourceBuilder());
@@ -229,9 +257,17 @@ public static void parseSearchRequest(
         if (searchRequest.pointInTimeBuilder() != null) {
             preparePointInTime(searchRequest, request);
         } else {
-            searchRequest.setCcsMinimizeRoundtrips(
-                request.paramAsBoolean("ccs_minimize_roundtrips", searchRequest.isCcsMinimizeRoundtrips())
-            );
+            if (inCpsContext.orElse(false)) {
+                // We're in CPS environment. MRT should not be settable by the user.
+                if (request.hasParam("ccs_minimize_roundtrips")) {
+                    throw new IllegalArgumentException("Setting ccs_minimize_roundtrips is not supported in cross-project search context");
+                }
+            } else {
+                // Either we're in non-CPS environment or the endpoint isn't CPS enabled, so parse what's in the request.
+                searchRequest.setCcsMinimizeRoundtrips(
+                    request.paramAsBoolean("ccs_minimize_roundtrips", searchRequest.isCcsMinimizeRoundtrips())
+                );
+            }
         }
         if (request.paramAsBoolean("force_synthetic_source", false)) {
             searchRequest.setForceSyntheticSource(true);

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.lookup.SourceFilter;
 
+import java.util.HashMap;
 import java.util.Map;
 
 public final class FetchSourcePhase implements FetchSubPhase {
@@ -99,6 +100,10 @@ private Source replaceInferenceMetadataFields(SearchHit hit, Source source) {
                     return source;
                 }
                 var newSource = source.source();
+                if (newSource instanceof HashMap == false) {
+                    // the map is not mutable
+                    newSource = new HashMap<>(newSource);
+                }
                 newSource.put(InferenceMetadataFieldsMapper.NAME, field.getValues().get(0));
                 return Source.fromMap(newSource, source.sourceContentType());
             }