Merge branch 'failure-store-security-poc' of github.com:n1v0lg/elasticsearch into failure-store-security-poc

Author: n1v0lg

.buildkite/pipelines/periodic.template.yml

@@ -86,6 +86,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
+              - openjdk24
             GRADLE_TASK:
               - checkPart1
               - checkPart2

.buildkite/pipelines/periodic.yml

@@ -505,6 +505,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
+              - openjdk24
             GRADLE_TASK:
               - checkPart1
               - checkPart2

docs/changelog/120751.yaml

@@ -0,0 +1,5 @@
+pr: 120751
+summary: Adding support for binary embedding type to Cohere service embedding type
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/120957.yaml

@@ -0,0 +1,5 @@
+pr: 120957
+summary: Introduce `AllocationBalancingRoundSummaryService`
+area: Allocation
+type: enhancement
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -398,7 +398,9 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
         var pluginName = pluginResolver.apply(requestingClass);
         if (pluginName != null) {
             var pluginEntitlements = pluginsEntitlements.get(pluginName);
-            if (pluginEntitlements != null) {
+            if (pluginEntitlements == null) {
+                return ModuleEntitlements.NONE;
+            } else {
                 final String scopeName;
                 if (requestingModule.isNamed() == false) {
                     scopeName = ALL_UNNAMED;

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java

@@ -271,7 +271,7 @@ public void testAgentsEntitlements() throws IOException, ClassNotFoundException
             createEmptyTestServerPolicy(),
             List.of(new CreateClassLoaderEntitlement()),
             Map.of(),
-            c -> "test",
+            c -> c.getPackageName().startsWith(TEST_AGENTS_PACKAGE_NAME) ? null : "test",
             TEST_AGENTS_PACKAGE_NAME,
             NO_ENTITLEMENTS_MODULE
         );
@@ -357,6 +357,22 @@ public void testDuplicateFlagEntitlements() {
         );
     }
 
+    /**
+     * If the plugin resolver tells us a class is in a plugin, don't conclude that it's in an agent.
+     */
+    public void testPluginResolverOverridesAgents() {
+        var policyManager = new PolicyManager(
+            createEmptyTestServerPolicy(),
+            List.of(new CreateClassLoaderEntitlement()),
+            Map.of(),
+            c -> "test", // Insist that the class is in a plugin
+            TEST_AGENTS_PACKAGE_NAME,
+            NO_ENTITLEMENTS_MODULE
+        );
+        ModuleEntitlements notAgentsEntitlements = policyManager.getEntitlements(TestAgent.class);
+        assertThat(notAgentsEntitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(false));
+    }
+
     private static Class<?> makeClassInItsOwnModule() throws IOException, ClassNotFoundException {
         final Path home = createTempDir();
         Path jar = createMockPluginJar(home);

muted-tests.yml

@@ -46,9 +46,6 @@ tests:
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test81JavaOptsInJvmOptions
   issue: https://github.com/elastic/elasticsearch/issues/113313
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=mtermvectors/10_basic/Tests catching other exceptions per item}
-  issue: https://github.com/elastic/elasticsearch/issues/113325
 - class: org.elasticsearch.xpack.transform.integration.TransformIT
   method: testStopWaitForCheckpoint
   issue: https://github.com/elastic/elasticsearch/issues/106113
@@ -449,6 +446,9 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=cat.aliases/10_basic/Complex alias}
   issue: https://github.com/elastic/elasticsearch/issues/121513
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=snapshot.create/10_basic/Create a snapshot for missing index}
+  issue: https://github.com/elastic/elasticsearch/issues/121536
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mtermvectors/10_basic.yml

@@ -3,6 +3,7 @@ setup:
   - do:
       indices.create:
           index: testidx
+          wait_for_active_shards: all
           body:
             mappings:
               properties:
@@ -80,6 +81,7 @@ setup:
   - do:
       indices.create:
           index: testidx2
+          wait_for_active_shards: all
 
   - do:
       indices.put_alias:

server/src/main/java/org/elasticsearch/ReleaseVersions.java

@@ -78,10 +78,10 @@ public static IntFunction<String> generateVersionsLookup(Class<?> versionContain
             // replace all version lists with the smallest & greatest versions
             versions.replaceAll((k, v) -> {
                 if (v.size() == 1) {
-                    return List.of(v.get(0));
+                    return List.of(v.getFirst());
                 } else {
                     v.sort(Comparator.naturalOrder());
-                    return List.of(v.get(0), v.get(v.size() - 1));
+                    return List.of(v.getFirst(), v.getLast());
                 }
             });
 
@@ -100,14 +100,14 @@ private static IntFunction<String> lookupFunction(NavigableMap<Integer, List<Ver
 
             String lowerBound, upperBound;
             if (versionRange != null) {
-                lowerBound = versionRange.get(0).toString();
-                upperBound = lastItem(versionRange).toString();
+                lowerBound = versionRange.getFirst().toString();
+                upperBound = versionRange.getLast().toString();
             } else {
                 // infer the bounds from the surrounding entries
                 var lowerRange = versions.lowerEntry(id);
                 if (lowerRange != null) {
                     // the next version is just a guess - might be a newer revision, might be a newer minor or major...
-                    lowerBound = nextVersion(lastItem(lowerRange.getValue())).toString();
+                    lowerBound = nextVersion(lowerRange.getValue().getLast()).toString();
                 } else {
                     // a really old version we don't have a record for
                     // assume it's an old version id - we can just return it directly
@@ -122,7 +122,7 @@ private static IntFunction<String> lookupFunction(NavigableMap<Integer, List<Ver
                 var upperRange = versions.higherEntry(id);
                 if (upperRange != null) {
                     // too hard to guess what version this id might be for using the next version - just use it directly
-                    upperBound = upperRange.getValue().get(0).toString();
+                    upperBound = upperRange.getValue().getFirst().toString();
                 } else {
                     // a newer version than all we know about? Can't map it...
                     upperBound = "[" + id + "]";
@@ -133,10 +133,6 @@ private static IntFunction<String> lookupFunction(NavigableMap<Integer, List<Ver
         };
     }
 
-    private static <T> T lastItem(List<T> list) {
-        return list.get(list.size() - 1);
-    }
-
     private static Version nextVersion(Version version) {
         return new Version(version.id + 100);   // +1 to revision
     }

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -173,7 +173,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_0_00);
     public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00);
     public static final TransportVersion ELASTICSEARCH_9_0 = def(9_000_0_00);
-
+    public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _