Merge branch 'main' into fix-semantic-query-rewrite-boost-issue

Author: elasticmachine

docs/changelog/128291.yaml

@@ -0,0 +1,5 @@
+pr: 128291
+summary: Make `dense_vector` fields updatable to bbq_flat/bbq_hnsw
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/129200.yaml

@@ -0,0 +1,5 @@
+pr: 129200
+summary: Simplified Linear Retriever
+area: Search
+type: enhancement
+issues: []

docs/changelog/129440.yaml

@@ -0,0 +1,5 @@
+pr: 129440
+summary: Fix filtered knn vector search when query timeouts are enabled
+area: Vector Search
+type: bug
+issues: []

docs/reference/elasticsearch/mapping-reference/dense-vector.md

@@ -397,10 +397,10 @@ POST /my-bit-vectors/_search?filter_path=hits.hits
 To better accommodate scaling and performance needs, updating the `type` setting in `index_options` is possible with the [Update Mapping API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-put-mapping), according to the following graph (jumps allowed):
 
 ```txt
-flat --> int8_flat --> int4_flat --> hnsw --> int8_hnsw --> int4_hnsw
+flat --> int8_flat --> int4_flat --> bbq_flat --> hnsw --> int8_hnsw --> int4_hnsw --> bbq_hnsw
 ```
 
-For updating all HNSW types (`hnsw`, `int8_hnsw`, `int4_hnsw`) the number of connections `m` must either stay the same or increase. For scalar quantized formats  (`int8_flat`, `int4_flat`, `int8_hnsw`, `int4_hnsw`) the `confidence_interval` must always be consistent (once defined, it cannot change).
+For updating all HNSW types (`hnsw`, `int8_hnsw`, `int4_hnsw`, `bbq_hnsw`) the number of connections `m` must either stay the same or increase. For the scalar quantized formats  `int8_flat`, `int4_flat`, `int8_hnsw` and `int4_hnsw` the `confidence_interval` must always be consistent (once defined, it cannot change).
 
 Updating `type` in `index_options` will fail in all other scenarios.
 

muted-tests.yml

@@ -271,9 +271,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test schedule_now on an already started transform}
   issue: https://github.com/elastic/elasticsearch/issues/120720
-- class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceTests
-  method: testIORateIsAdjustedForRunningMergeTasks
-  issue: https://github.com/elastic/elasticsearch/issues/125842
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Verify start transform creates destination index with appropriate mapping}
   issue: https://github.com/elastic/elasticsearch/issues/125854
@@ -301,9 +298,6 @@ tests:
 - class: org.elasticsearch.index.engine.ThreadPoolMergeSchedulerTests
   method: testSchedulerCloseWaitsForRunningMerge
   issue: https://github.com/elastic/elasticsearch/issues/125236
-- class: org.elasticsearch.xpack.security.SecurityRolesMultiProjectIT
-  method: testUpdatingFileBasedRoleAffectsAllProjects
-  issue: https://github.com/elastic/elasticsearch/issues/126223
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test020PluginsListWithNoPlugins
   issue: https://github.com/elastic/elasticsearch/issues/126232
@@ -565,6 +559,9 @@ tests:
 - class: org.elasticsearch.xpack.ilm.actions.ShrinkActionIT
   method: testShrinkDuringSnapshot
   issue: https://github.com/elastic/elasticsearch/issues/129491
+- class: org.elasticsearch.xpack.security.PermissionsIT
+  method: testWhenUserLimitedByOnlyAliasOfIndexCanWriteToIndexWhichWasRolledoverByILMPolicy
+  issue: https://github.com/elastic/elasticsearch/issues/129481
 
 # Examples:
 #

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 
@@ -31,6 +32,8 @@
  */
 public class DenseVectorMappingUpdateIT extends AbstractRollingUpgradeTestCase {
 
+    private static final String SYNTHETIC_SOURCE_FEATURE = "gte_v8.12.0";
+
     private static final String BULK1 = """
                     {"index": {"_id": "1"}}
                     {"embedding": [1, 1, 1, 1]}
@@ -86,10 +89,22 @@ public void testDenseVectorMappingUpdateOnOldCluster() throws IOException {
             String indexName = "test_index";
             if (isOldCluster()) {
                 Request createIndex = new Request("PUT", "/" + indexName);
-                XContentBuilder mappings = XContentBuilder.builder(XContentType.JSON.xContent())
-                    .startObject()
-                    .startObject("mappings")
-                    .startObject("properties")
+                boolean useSyntheticSource = randomBoolean() && oldClusterHasFeature(SYNTHETIC_SOURCE_FEATURE);
+
+                boolean useIndexSetting = SourceFieldMapper.onOrAfterDeprecateModeVersion(getOldClusterIndexVersion());
+                XContentBuilder payload = XContentBuilder.builder(XContentType.JSON.xContent()).startObject();
+                if (useSyntheticSource) {
+                    if (useIndexSetting) {
+                        payload.startObject("settings").field("index.mapping.source.mode", "synthetic").endObject();
+                    }
+                }
+                payload.startObject("mappings");
+                if (useIndexSetting == false) {
+                    payload.startObject("_source");
+                    payload.field("mode", "synthetic");
+                    payload.endObject();
+                }
+                payload.startObject("properties")
                     .startObject("embedding")
                     .field("type", "dense_vector")
                     .field("index", "true")
@@ -104,7 +119,7 @@ public void testDenseVectorMappingUpdateOnOldCluster() throws IOException {
                     .endObject()
                     .endObject()
                     .endObject();
-                createIndex.setJsonEntity(Strings.toString(mappings));
+                createIndex.setJsonEntity(Strings.toString(payload));
                 client().performRequest(createIndex);
                 Request index = new Request("POST", "/" + indexName + "/_bulk/");
                 index.addParameter("refresh", "true");