Merge branch 'main' into ml-async-sender-init

Author: jonathan-buttner

docs/changelog/130485.yaml

@@ -0,0 +1,5 @@
+pr: 130485
+summary: Add `RerankRequestChunker`
+area: Machine Learning
+type: enhancement
+issues: []

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -698,19 +698,13 @@ public void testAddDimensionToMapping() throws Exception {
             }
             """, XContentType.JSON);
         ActionFuture<AcknowledgedResponse> putMappingFuture = client().execute(TransportPutMappingAction.TYPE, putMappingRequest);
-        if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
-            IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, putMappingFuture::actionGet);
-            assertThat(
-                exception.getMessage(),
-                containsString("Cannot add dynamic templates that define dimension fields on an existing index with index.dimensions")
-            );
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name"));
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
-        } else {
-            assertAcked(putMappingFuture);
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), containsInAnyOrder("metricset"));
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty());
-        }
+        IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, putMappingFuture::actionGet);
+        assertThat(
+            exception.getMessage(),
+            containsString("Cannot add dynamic templates that define dimension fields on an existing index with index.dimensions")
+        );
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name"));
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
         indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat");
     }
 

server/src/main/resources/transport/definitions/referable/elastic_reranker_chunking_configuration.csv

@@ -0,0 +1 @@
+9180000

server/src/main/resources/transport/upper_bounds/9.2.csv

@@ -1 +1 @@
-ml_inference_google_model_garden_added,9179000
+elastic_reranker_chunking_configuration,9180000

test/framework/build.gradle

@@ -10,29 +10,29 @@
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
 
-configurations {
-  // we do not want to expose a version conflict in transitive dependencies by
-  // bringing in different versions of hamcrest and hamcrest-core.
-  // Therefore we exclude transitive deps on hamcrest-core here as we have a direct
-  // dependency on a newer version.
-  runtimeElements {
-    exclude group: 'org.hamcrest', module: 'hamcrest-core'
-  }
-}
 dependencies {
   api project(":client:rest")
   api project(':modules:transport-netty4')
   api project(':libs:ssl-config')
   api project(":server")
   api project(":libs:cli")
   api project(":libs:entitlement:bridge")
-  api "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
-  api("junit:junit:${versions.junit}") {
-//    exclude group: 'org.hamcrest'
+  api ("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}") {
+    exclude group: "junit", module: "junit"
   }
+
+  // we do not want to expose a version conflict in transitive dependencies by
+  // bringing in different versions of dependencies.
+  // Therefore we exclude mismatches in our transitive dependencies explicitly.
+  // This also avoids jarHell issues with different hamcrest related dependencies
+  // like hamcrest and hamcrest-core
   api "org.hamcrest:hamcrest:${versions.hamcrest}"
+  api("junit:junit:${versions.junit}") {
+    exclude group: 'org.hamcrest', module: 'hamcrest-core'
+  }
   api("org.apache.lucene:lucene-test-framework:${versions.lucene}") {
-//    exclude group: 'org.hamcrest'
+    exclude group: "junit", module: "junit"
+    exclude group: 'com.carrotsearch.randomizedtesting', module: 'randomizedtesting-runner'
   }
   api "org.apache.lucene:lucene-codecs:${versions.lucene}"
   api "commons-logging:commons-logging:${versions.commonslogging}"

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java

@@ -25,7 +25,8 @@ public enum FeatureFlag {
         "es.index_dimensions_tsid_optimization_feature_flag_enabled=true",
         Version.fromString("9.2.0"),
         null
-    );
+    ),
+    ELASTIC_RERANKER_CHUNKING("es.elastic_reranker_chunking_long_documents=true", Version.fromString("9.2.0"), null);
 
     public final String systemProperty;
     public final Version from;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java

@@ -38,7 +38,6 @@
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -316,7 +315,7 @@ public Map<String, Object> usageStats() {
         stats.put("evictions", cacheStats.getEvictions());
         stats.put("hits_time_in_millis", TimeValue.nsecToMSec(hitsTimeInNanos.sum()));
         stats.put("misses_time_in_millis", TimeValue.nsecToMSec(missesTimeInNanos.sum()));
-        return Collections.unmodifiableMap(stats);
+        return stats;
     }
 
     private static final class BitsetCacheKey {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/stats/GetSecurityStatsNodeResponseTests.java

@@ -9,12 +9,15 @@
 
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
 import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class GetSecurityStatsNodeResponseTests extends AbstractWireSerializingTestCase<GetSecurityStatsNodeResponse> {
 
     @Override
@@ -46,4 +49,20 @@ protected GetSecurityStatsNodeResponse mutateInstance(GetSecurityStatsNodeRespon
             default -> throw new IllegalStateException("Unexpected value");
         };
     }
+
+    public void testRolesStatsInOrderSerialization() throws IOException {
+        final Map<String, Object> rolesStats = Maps.newLinkedHashMapWithExpectedSize(3);
+        rolesStats.put("one", "value");
+        rolesStats.put("two", "value");
+        rolesStats.put("three", "value");
+        final GetSecurityStatsNodeResponse in = new GetSecurityStatsNodeResponse(
+            DiscoveryNodeUtils.create(randomAlphaOfLength(10)),
+            rolesStats
+        );
+
+        final GetSecurityStatsNodeResponse out = copyInstance(in);
+        assertThat(in, equalTo(out));
+
+        assertThat(out.getRolesStoreStats().keySet().toArray(new String[0]), equalTo(new String[] { "one", "two", "three" }));
+    }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java

@@ -65,6 +65,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
@@ -529,6 +530,11 @@ public void testHitsMissesAndEvictionsStats() throws Exception {
         assertThat(cache.usageStats(), equalTo(finalStats));
     }
 
+    public void testUsageStatsAreOrdered() {
+        final Map<String, Object> stats = newCache(Settings.EMPTY).usageStats();
+        assertThat("needs to be LinkedHashMap for order in transport", stats, instanceOf(LinkedHashMap.class));
+    }
+
     private void runTestOnIndex(CheckedBiConsumer<SearchExecutionContext, LeafReaderContext, Exception> body) throws Exception {
         runTestOnIndices(1, ctx -> {
             final TestIndexContext indexContext = ctx.get(0);

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java

@@ -17,6 +17,9 @@ public class ChunkingSettingsBuilder {
     public static final SentenceBoundaryChunkingSettings DEFAULT_SETTINGS = new SentenceBoundaryChunkingSettings(250, 1);
     // Old settings used for backward compatibility for endpoints created before 8.16 when default was changed
     public static final WordBoundaryChunkingSettings OLD_DEFAULT_SETTINGS = new WordBoundaryChunkingSettings(250, 100);
+    public static final int ELASTIC_RERANKER_TOKEN_LIMIT = 512;
+    public static final int ELASTIC_RERANKER_EXTRA_TOKEN_COUNT = 3;
+    public static final float WORDS_PER_TOKEN = 0.75f;
 
     public static ChunkingSettings fromMap(Map<String, Object> settings) {
         return fromMap(settings, true);
@@ -51,4 +54,17 @@ public static ChunkingSettings fromMap(Map<String, Object> settings, boolean ret
             case RECURSIVE -> RecursiveChunkingSettings.fromMap(new HashMap<>(settings));
         };
     }
+
+    public static ChunkingSettings buildChunkingSettingsForElasticRerank(int queryWordCount) {
+        var queryTokenCount = Math.ceil(queryWordCount / WORDS_PER_TOKEN);
+        var chunkSizeTokenCountWithFullQuery = (ELASTIC_RERANKER_TOKEN_LIMIT - ELASTIC_RERANKER_EXTRA_TOKEN_COUNT - queryTokenCount);
+
+        var maxChunkSizeTokenCount = Math.floor((float) ELASTIC_RERANKER_TOKEN_LIMIT / 2);
+        if (chunkSizeTokenCountWithFullQuery > maxChunkSizeTokenCount) {
+            maxChunkSizeTokenCount = chunkSizeTokenCountWithFullQuery;
+        }
+
+        var maxChunkSizeWordCount = (int) (maxChunkSizeTokenCount * WORDS_PER_TOKEN);
+        return new SentenceBoundaryChunkingSettings(maxChunkSizeWordCount, 1);
+    }
 }