Merge branch 'refs/heads/main' into carlosdelest/fix-synonyms-ci-tests-timeout

Author: carlosdelest

docs/changelog/112929.yaml

@@ -0,0 +1,5 @@
+pr: 112929
+summary: "ES|QL: Add support for cached strings in plan serialization"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/112938.yaml

@@ -0,0 +1,35 @@
+pr: 112938
+summary: Enhance SORT push-down to Lucene to cover references to fields and ST_DISTANCE function
+area: ES|QL
+type: enhancement
+issues:
+ - 109973
+highlight:
+  title: Enhance SORT push-down to Lucene to cover references to fields and ST_DISTANCE function
+  body: |-
+    The most used and likely most valuable geospatial search query in Elasticsearch is the sorted proximity search,
+    finding items within a certain distance of a point of interest and sorting the results by distance.
+    This has been possible in ES|QL since 8.15.0, but the sorting was done in-memory, not pushed down to Lucene.
+    Now the sorting is pushed down to Lucene, which results in a significant performance improvement.
+
+    Queries that perform both filtering and sorting on distance are supported. For example:
+
+    [source,esql]
+    ----
+    FROM test
+    | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(37.7749, -122.4194)"))
+    | WHERE distance < 1000000
+    | SORT distance ASC, name DESC
+    | LIMIT 10
+    ----
+
+    In addition, the support for sorting on EVAL expressions has been extended to cover references to fields:
+
+    [source,esql]
+    ----
+    FROM test
+    | EVAL ref = field
+    | SORT ref ASC
+    | LIMIT 10
+    ----
+  notable: false

docs/changelog/114411.yaml

@@ -0,0 +1,5 @@
+pr: 114411
+summary: "ESQL: Push down filters even in case of renames in Evals"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/114533.yaml

@@ -0,0 +1,5 @@
+pr: 114533
+summary: Fix dim validation for bit `element_type`
+area: Vector Search
+type: bug
+issues: []

docs/changelog/114552.yaml

@@ -0,0 +1,5 @@
+pr: 114552
+summary: Improve exception message for bad environment variable placeholders in settings
+area: Infra/Settings
+type: enhancement
+issues: [110858]

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java

@@ -34,7 +34,26 @@ private DeleteDataStreamLifecycleAction() {/* no instances */}
     public static final class Request extends AcknowledgedRequest<Request> implements IndicesRequest.Replaceable {
 
         private String[] names;
-        private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
+        private IndicesOptions indicesOptions = IndicesOptions.builder()
+            .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+            .wildcardOptions(
+                IndicesOptions.WildcardOptions.builder()
+                    .matchOpen(true)
+                    .matchClosed(true)
+                    .includeHidden(false)
+                    .resolveAliases(false)
+                    .allowEmptyExpressions(true)
+                    .build()
+            )
+            .gatekeeperOptions(
+                IndicesOptions.GatekeeperOptions.builder()
+                    .allowAliasToMultipleIndices(false)
+                    .allowClosedIndices(true)
+                    .ignoreThrottled(false)
+                    .allowFailureIndices(false)
+                    .build()
+            )
+            .build();
 
         public Request(StreamInput in) throws IOException {
             super(in);

muted-tests.yml

@@ -71,9 +71,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=rollup/security_tests/Index-based access}
   issue: https://github.com/elastic/elasticsearch/issues/111631
-- class: org.elasticsearch.tdigest.ComparisonTests
-  method: testSparseGaussianDistribution
-  issue: https://github.com/elastic/elasticsearch/issues/111721
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
   method: testSnapshotRestore {cluster=OLD}
   issue: https://github.com/elastic/elasticsearch/issues/111777
@@ -343,9 +340,6 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search}
   issue: https://github.com/elastic/elasticsearch/issues/114371
-- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT
-  method: testProfileOrdinalsGroupingOperator {SYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/114380
 - class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests
   method: testInfer_StreamRequest
   issue: https://github.com/elastic/elasticsearch/issues/114385
@@ -384,6 +378,15 @@ tests:
 - class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
   method: testTermsQuery
   issue: https://github.com/elastic/elasticsearch/issues/114563
+- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
+  method: testMatchAllQuery
+  issue: https://github.com/elastic/elasticsearch/issues/114607
+- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests
+  method: testPushSpatialIntersectsEvalToSource {default}
+  issue: https://github.com/elastic/elasticsearch/issues/114627
+- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests
+  method: testPushWhereEvalToSource {default}
+  issue: https://github.com/elastic/elasticsearch/issues/114628
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml

@@ -21,13 +21,13 @@ setup:
                 element_type: byte
                 index_options:
                   type: hnsw
+                  m: 16
+                  ef_construction: 100
               vector2:
                 type: dense_vector
                 dims: 1024
                 index: true
                 similarity: dot_product
-                index_options:
-                  type: int8_hnsw
               vector3:
                 type: dense_vector
                 dims: 100

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -239,6 +239,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion TEXT_SIMILARITY_RERANKER_QUERY_REWRITE = def(8_763_00_0);
     public static final TransportVersion SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS = def(8_764_00_0);
     public static final TransportVersion RETRIEVERS_TELEMETRY_ADDED = def(8_765_00_0);
+    public static final TransportVersion ESQL_CACHED_STRING_SERIALIZATION = def(8_766_00_0);
+    public static final TransportVersion CHUNK_SENTENCE_OVERLAP_SETTING_ADDED = def(8_767_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java

@@ -58,7 +58,26 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
     // indices options that require every specified index to exist, expand wildcards only to open
     // indices, don't allow that no indices are resolved from wildcard expressions and resolve the
     // expressions only against indices
-    private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
+    private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.builder()
+        .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            IndicesOptions.WildcardOptions.builder()
+                .matchOpen(true)
+                .matchClosed(false)
+                .includeHidden(false)
+                .resolveAliases(false)
+                .allowEmptyExpressions(false)
+                .build()
+        )
+        .gatekeeperOptions(
+            IndicesOptions.GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(true)
+                .allowClosedIndices(true)
+                .ignoreThrottled(false)
+                .allowFailureIndices(true)
+                .build()
+        )
+        .build();
 
     public IndicesAliasesRequest(StreamInput in) throws IOException {
         super(in);