Merge remote-tracking branch 'es/main' into fix_seqno_field_use_tsdb_doc_values_format_ff

Author: martijnvg

build-tools/src/main/java/org/elasticsearch/gradle/VersionProperties.java

@@ -17,8 +17,8 @@
 /**
  * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
  *
- * @deprecated use ext values set by {@link org.elasticsearch.gradle.internal.conventions.VersionPropertiesPlugin} or
- * {@link org.elasticsearch.gradle.internal.conventions.VersionPropertiesBuildService}
+ * @deprecated use ext values set by org.elasticsearch.gradle.internal.conventions.VersionPropertiesPlugin or
+ * org.elasticsearch.gradle.internal.conventions.VersionPropertiesBuildService
  *
  */
 @Deprecated

docs/changelog/133919.yaml

@@ -0,0 +1,5 @@
+pr: 133919
+summary: Fix double-counting of inference memory in the assignment rebalancer
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/133930.yaml

@@ -0,0 +1,6 @@
+pr: 133930
+summary: Improve memory estimation methods accuracy in `TrainedModelAssignmentRebalancer`
+  and related classes
+area: Machine Learning
+type: bug
+issues: []

muted-tests.yml

@@ -537,6 +537,12 @@ tests:
 - class: org.elasticsearch.xpack.esql.action.RandomizedTimeSeriesIT
   method: testRateGroupBySubset
   issue: https://github.com/elastic/elasticsearch/issues/134019
+- class: org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDeciderIT
+  method: testDesiredNodesAreTakenIntoAccountInAutoExpandReplicas
+  issue: https://github.com/elastic/elasticsearch/issues/134049
+- class: org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDeciderIT
+  method: testShardsAreKeptInPreferredTierUntilTheNextTierIsInItsFinalState
+  issue: https://github.com/elastic/elasticsearch/issues/134050
 
 # Examples:
 #

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java

@@ -16,7 +16,7 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationFailureHandler;
 import org.elasticsearch.xpack.core.security.authc.Realm;
-import org.elasticsearch.xpack.core.security.authc.apikey.CustomApiKeyAuthenticator;
+import org.elasticsearch.xpack.core.security.authc.apikey.CustomAuthenticator;
 import org.elasticsearch.xpack.core.security.authc.service.NodeLocalServiceAccountTokenStore;
 import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountTokenStore;
 import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
@@ -129,7 +129,7 @@ default ServiceAccountTokenStore getServiceAccountTokenStore(SecurityComponents
         return null;
     }
 
-    default CustomApiKeyAuthenticator getCustomApiKeyAuthenticator(SecurityComponents components) {
+    default List<CustomAuthenticator> getCustomAuthenticators(SecurityComponents components) {
         return null;
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java

@@ -1376,6 +1376,18 @@ public static Authentication newRealmAuthentication(User user, RealmRef realmRef
         return authentication;
     }
 
+    public static Authentication newCloudAccessTokenAuthentication(
+        AuthenticationResult<User> authResult,
+        Authentication.RealmRef realmRef
+    ) {
+        assert authResult.isAuthenticated() : "cloud access token authn result must be successful";
+        final User user = authResult.getValue();
+        return new Authentication(
+            new Subject(user, realmRef, TransportVersion.current(), authResult.getMetadata()),
+            AuthenticationType.TOKEN
+        );
+    }
+
     public static Authentication newCloudApiKeyAuthentication(AuthenticationResult<User> authResult, String nodeName) {
         assert authResult.isAuthenticated() : "cloud API Key authn result must be successful";
         final User apiKeyUser = authResult.getValue();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/apikey/CustomApiKeyAuthenticator.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.authc.apikey;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
+
+/**
+ * An extension point to provide a custom authenticator implementation. For example, a custom API key or a custom OAuth2
+ * token implementation. The implementation is wrapped by a core `Authenticator` class and included in the authenticator chain
+ * _before_ the respective "standard" authenticator(s).
+ */
+public interface CustomAuthenticator {
+
+    boolean supports(AuthenticationToken token);
+
+    @Nullable
+    AuthenticationToken extractToken(ThreadContext context);
+
+    void authenticate(@Nullable AuthenticationToken token, ActionListener<AuthenticationResult<Authentication>> listener);
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/apikey/CustomAuthenticator.java

@@ -56,7 +56,7 @@ public void testSetDuration_GivenZero() {
     }
 
     public void testSetDuration_GivenNegative() {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new Request().setDuration("-1s"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new Request().setDuration("-1"));
         assertThat(e.getMessage(), equalTo("[duration] must be positive: [-1]"));
     }
 
@@ -67,7 +67,7 @@ public void testSetExpiresIn_GivenZero() {
     }
 
     public void testSetExpiresIn_GivenNegative() {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new Request().setExpiresIn("-1s"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new Request().setExpiresIn("-1"));
         assertThat(e.getMessage(), equalTo("[expires_in] must be non-negative: [-1]"));
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ForecastJobActionRequestTests.java

@@ -298,9 +298,7 @@ private Map<List<String>, List<AssignmentPlan.Node>> createNodesByZoneMap() {
                         nodes.add(
                             new AssignmentPlan.Node(
                                 discoveryNode.getId(),
-                                // We subtract native inference memory as the planner expects available memory for
-                                // native inference including current assignments.
-                                getNodeFreeMemoryExcludingPerNodeOverheadAndNativeInference(load),
+                                load.getFreeMemoryExcludingPerNodeOverhead(),
                                 MlProcessors.get(discoveryNode, allocatedProcessorsScale).roundUp()
                             )
                         );
@@ -317,10 +315,6 @@ private Map<List<String>, List<AssignmentPlan.Node>> createNodesByZoneMap() {
         }));
     }
 
-    private static long getNodeFreeMemoryExcludingPerNodeOverheadAndNativeInference(NodeLoad load) {
-        return load.getFreeMemoryExcludingPerNodeOverhead() - load.getAssignedNativeInferenceMemory();
-    }
-
     private TrainedModelAssignmentMetadata.Builder buildAssignmentsFromPlan(AssignmentPlan assignmentPlan) {
         TrainedModelAssignmentMetadata.Builder builder = TrainedModelAssignmentMetadata.Builder.empty();
         for (AssignmentPlan.Deployment deployment : assignmentPlan.deployments()) {
@@ -406,7 +400,28 @@ private Optional<String> explainAssignment(
             return Optional.of(load.getError());
         }
 
-        if (deployment.memoryBytes() > assignmentPlan.getRemainingNodeMemory(node.getId())) {
+        // Find how many allocations already exist on this node
+        // We need to search by node ID as assignmentPlan.assignments() returns a map
+        // of AssignmentPlan.Node and the argument node of the DiscoveryNode
+        int existingAllocationsOnNode = assignmentPlan.assignments(deployment)
+            .map(
+                assignments -> assignments.getOrDefault(
+                    assignments.keySet().stream().filter(n -> n.id().equals(node.getId())).findFirst().orElse(null),
+                    0
+                )
+            )
+            .orElse(0);
+
+        // Calculate how many allocations remain to be assigned
+        int unassignedAllocations = deployment.allocations() - assignmentPlan.totalAllocations(deployment);
+
+        // Check if there's enough memory for additional allocations
+        long additionalMemory = deployment.estimateAdditionalMemoryUsageBytes(
+            existingAllocationsOnNode,
+            existingAllocationsOnNode + unassignedAllocations
+        );
+        long availableMemory = assignmentPlan.getRemainingNodeMemory(node.getId());
+        if (additionalMemory > availableMemory) {
             // If any ML processes are running on a node we require some space to load the shared libraries.
             // So if none are currently running then this per-node overhead must be added to the requirement.
             // From node load we know if we had any jobs or models assigned before the rebalance.