Merge branch 'main' into fuse_out_of_snapshot

Author: ioanatia

docs/changelog/130485.yaml

@@ -0,0 +1,5 @@
+pr: 130485
+summary: Add `RerankRequestChunker`
+area: Machine Learning
+type: enhancement
+issues: []

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -698,19 +698,13 @@ public void testAddDimensionToMapping() throws Exception {
             }
             """, XContentType.JSON);
         ActionFuture<AcknowledgedResponse> putMappingFuture = client().execute(TransportPutMappingAction.TYPE, putMappingRequest);
-        if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
-            IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, putMappingFuture::actionGet);
-            assertThat(
-                exception.getMessage(),
-                containsString("Cannot add dynamic templates that define dimension fields on an existing index with index.dimensions")
-            );
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name"));
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
-        } else {
-            assertAcked(putMappingFuture);
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), containsInAnyOrder("metricset"));
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty());
-        }
+        IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, putMappingFuture::actionGet);
+        assertThat(
+            exception.getMessage(),
+            containsString("Cannot add dynamic templates that define dimension fields on an existing index with index.dimensions")
+        );
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name"));
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
         indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat");
     }
 

muted-tests.yml

@@ -594,18 +594,12 @@ tests:
 - class: org.elasticsearch.action.admin.cluster.stats.SearchUsageStatsTests
   method: testToXContent
   issue: https://github.com/elastic/elasticsearch/issues/135558
-- class: org.elasticsearch.search.ccs.SparseVectorQueryBuilderCrossClusterSearchIT
-  method: testSparseVectorQueryWithCcsMinimizeRoundTripsFalse
-  issue: https://github.com/elastic/elasticsearch/issues/135559
 - class: org.elasticsearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDeciderTests
   method: testCanAllocatePrimaryExistingInRestoreInProgress
   issue: https://github.com/elastic/elasticsearch/issues/135566
 - class: org.elasticsearch.xpack.esql.inference.textembedding.TextEmbeddingOperatorTests
   method: testSimpleCircuitBreaking
   issue: https://github.com/elastic/elasticsearch/issues/135569
-- class: org.elasticsearch.search.ccs.KnnVectorQueryBuilderCrossClusterSearchIT
-  method: testKnnQueryWithCcsMinimizeRoundTripsFalse
-  issue: https://github.com/elastic/elasticsearch/issues/135573
 - class: org.elasticsearch.multiproject.test.XpackWithMultipleProjectsClientYamlTestSuiteIT
   method: test {yaml=esql/60_usage/Basic ESQL usage output (telemetry) snapshot version}
   issue: https://github.com/elastic/elasticsearch/issues/135579

server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java

@@ -38,7 +38,10 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
     implements
         IndicesRequest {
 
+    // superseded
     private static final TransportVersion INDEX_RESHARD_SHARDCOUNT_SUMMARY = TransportVersion.fromName("index_reshard_shardcount_summary");
+    // bumped to use VInt instead of Int
+    private static final TransportVersion INDEX_RESHARD_SHARDCOUNT_SMALL = TransportVersion.fromName("index_reshard_shardcount_small");
 
     public static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueMinutes(1);
 
@@ -126,14 +129,16 @@ public ReplicationRequest(@Nullable ShardId shardId, int reshardSplitShardCountS
             index = in.readString();
         }
         routedBasedOnClusterVersion = in.readVLong();
-        if (in.getTransportVersion().supports(INDEX_RESHARD_SHARDCOUNT_SUMMARY)) {
-            if (thinRead) {
-                this.reshardSplitShardCountSummary = reshardSplitShardCountSummary;
-            } else {
+        if (thinRead) {
+            this.reshardSplitShardCountSummary = reshardSplitShardCountSummary;
+        } else {
+            if (in.getTransportVersion().supports(INDEX_RESHARD_SHARDCOUNT_SMALL)) {
+                this.reshardSplitShardCountSummary = in.readVInt();
+            } else if (in.getTransportVersion().supports(INDEX_RESHARD_SHARDCOUNT_SUMMARY)) {
                 this.reshardSplitShardCountSummary = in.readInt();
+            } else {
+                this.reshardSplitShardCountSummary = 0;
             }
-        } else {
-            this.reshardSplitShardCountSummary = 0;
         }
     }
 
@@ -262,7 +267,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeTimeValue(timeout);
         out.writeString(index);
         out.writeVLong(routedBasedOnClusterVersion);
-        if (out.getTransportVersion().supports(INDEX_RESHARD_SHARDCOUNT_SUMMARY)) {
+        if (out.getTransportVersion().supports(INDEX_RESHARD_SHARDCOUNT_SMALL)) {
+            out.writeVInt(reshardSplitShardCountSummary);
+        } else if (out.getTransportVersion().supports(INDEX_RESHARD_SHARDCOUNT_SUMMARY)) {
             out.writeInt(reshardSplitShardCountSummary);
         }
     }

server/src/main/resources/transport/definitions/referable/elastic_reranker_chunking_configuration.csv

@@ -0,0 +1 @@
+9180000

server/src/main/resources/transport/definitions/referable/index_reshard_shardcount_small.csv

@@ -0,0 +1 @@
+9181000

server/src/main/resources/transport/upper_bounds/9.2.csv

@@ -1 +1 @@
-ml_inference_google_model_garden_added,9179000
+index_reshard_shardcount_small,9181000

test/framework/build.gradle

@@ -10,29 +10,29 @@
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
 
-configurations {
-  // we do not want to expose a version conflict in transitive dependencies by
-  // bringing in different versions of hamcrest and hamcrest-core.
-  // Therefore we exclude transitive deps on hamcrest-core here as we have a direct
-  // dependency on a newer version.
-  runtimeElements {
-    exclude group: 'org.hamcrest', module: 'hamcrest-core'
-  }
-}
 dependencies {
   api project(":client:rest")
   api project(':modules:transport-netty4')
   api project(':libs:ssl-config')
   api project(":server")
   api project(":libs:cli")
   api project(":libs:entitlement:bridge")
-  api "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
-  api("junit:junit:${versions.junit}") {
-//    exclude group: 'org.hamcrest'
+  api ("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}") {
+    exclude group: "junit", module: "junit"
   }
+
+  // we do not want to expose a version conflict in transitive dependencies by
+  // bringing in different versions of dependencies.
+  // Therefore we exclude mismatches in our transitive dependencies explicitly.
+  // This also avoids jarHell issues with different hamcrest related dependencies
+  // like hamcrest and hamcrest-core
   api "org.hamcrest:hamcrest:${versions.hamcrest}"
+  api("junit:junit:${versions.junit}") {
+    exclude group: 'org.hamcrest', module: 'hamcrest-core'
+  }
   api("org.apache.lucene:lucene-test-framework:${versions.lucene}") {
-//    exclude group: 'org.hamcrest'
+    exclude group: "junit", module: "junit"
+    exclude group: 'com.carrotsearch.randomizedtesting', module: 'randomizedtesting-runner'
   }
   api "org.apache.lucene:lucene-codecs:${versions.lucene}"
   api "commons-logging:commons-logging:${versions.commonslogging}"

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java

@@ -25,7 +25,8 @@ public enum FeatureFlag {
         "es.index_dimensions_tsid_optimization_feature_flag_enabled=true",
         Version.fromString("9.2.0"),
         null
-    );
+    ),
+    ELASTIC_RERANKER_CHUNKING("es.elastic_reranker_chunking_long_documents=true", Version.fromString("9.2.0"), null);
 
     public final String systemProperty;
     public final Version from;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java

@@ -38,7 +38,6 @@
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -316,7 +315,7 @@ public Map<String, Object> usageStats() {
         stats.put("evictions", cacheStats.getEvictions());
         stats.put("hits_time_in_millis", TimeValue.nsecToMSec(hitsTimeInNanos.sum()));
         stats.put("misses_time_in_millis", TimeValue.nsecToMSec(missesTimeInNanos.sum()));
-        return Collections.unmodifiableMap(stats);
+        return stats;
     }
 
     private static final class BitsetCacheKey {