diff --git a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index d71718f3f3a6b..4ef8f5a86e7ee 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -34,6 +34,8 @@ import java.util.Objects; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFutureThrows; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -328,12 +330,22 @@ public void testUnassignRunningPersistentTask() throws Exception { // Disallow re-assignment after it is unassigned to verify master and node state TestPersistentTasksExecutor.setNonClusterStateCondition(false); - persistentTasksClusterService.unassignPersistentTask(taskId, task.getAllocationId() + 1, "unassignment test", unassignmentFuture); + logger.info("unassigning persistent task"); + persistentTasksClusterService.unassignPersistentTask( + taskId, + task.getAllocationId() + 1, + PersistentTasksCustomMetadata.Explanation.GENERIC_REASON, + "unassignment test", + unassignmentFuture + ); PersistentTask unassignedTask = unassignmentFuture.get(); assertThat(unassignedTask.getId(), equalTo(taskId)); - assertThat(unassignedTask.getAssignment().getExplanation(), equalTo("unassignment test")); + assertThat( + unassignedTask.getAssignment().getExplanationCodes(), + contains(PersistentTasksCustomMetadata.Explanation.GENERIC_REASON) + ); + assertThat(unassignedTask.getAssignment().getExplanation(), containsString("unassignment test")); assertThat(unassignedTask.getAssignment().getExecutorNode(), is(nullValue())); - assertBusy(() -> { // Verify that the task is NOT running on the node List tasks = clusterAdmin().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(); @@ -342,13 +354,14 @@ public void testUnassignRunningPersistentTask() throws Exception { // Verify that the task is STILL in internal cluster state assertClusterStateHasTask(taskId); }); + logger.info("persistent task unassigned"); // Allow it to be reassigned again to the same node TestPersistentTasksExecutor.setNonClusterStateCondition(true); - + logger.info("task is allowed to start again"); // Verify it starts again waitForTaskToStart(); - + logger.info("task has started again"); assertClusterStateHasTask(taskId); // Complete or cancel the running task @@ -406,6 +419,12 @@ public void testAbortLocally() throws Exception { task.getAssignment().getExplanation(), either(equalTo("Simulating local abort")).or(equalTo("non cluster state condition prevents assignment")) ); + assertThat( + task.getAssignment().getExplanationCodes(), + either(contains(PersistentTasksCustomMetadata.Explanation.ABORTED_LOCALLY)).or( + contains(PersistentTasksCustomMetadata.Explanation.GENERIC_REASON) + ) + ); }); // Allow it to be reassigned again @@ -420,7 +439,7 @@ public void testAbortLocally() throws Exception { // reason has not been published, hence the busy wait here.) assertBusy(() -> { PersistentTask task = assertClusterStateHasTask(taskId); - assertThat(task.getAssignment().getExplanation(), not(equalTo("Simulating local abort"))); + assertThat(task.getAssignment().getExplanation(), not(containsString("Simulating local abort"))); }); // Complete or cancel the running task diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index f740bdb6efb5d..5ae3f482c8245 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -212,6 +212,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS = def(8_703_00_0); public static final TransportVersion INFERENCE_ADAPTIVE_ALLOCATIONS = def(8_704_00_0); public static final TransportVersion INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN = def(8_705_00_0); + public static final TransportVersion PERSISTENT_TASK_CUSTOM_METADATA_ASSIGNMENT_REASON_ENUM = def(8_706_00_0); /* * STOP! READ THIS FIRST! No, really, @@ -276,7 +277,7 @@ static TransportVersion def(int id) { * Reference to the minimum transport version that can be used with CCS. * This should be the transport version used by the previous minor release. */ - public static final TransportVersion MINIMUM_CCS_VERSION = SHUTDOWN_REQUEST_TIMEOUTS_FIX_8_14; + public static final TransportVersion MINIMUM_CCS_VERSION = V_8_13_0; static final NavigableMap VERSION_IDS = getAllVersionIds(TransportVersions.class); diff --git a/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java b/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java index cc908cd7cad2c..ee63c90df8d27 100644 --- a/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java @@ -150,7 +150,10 @@ public PersistentTasksCustomMetadata.Assignment getAssignment( if (discoveryNode == null) { return NO_NODE_FOUND; } else { - return new PersistentTasksCustomMetadata.Assignment(discoveryNode.getId(), ""); + return new PersistentTasksCustomMetadata.Assignment( + discoveryNode.getId(), + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ); } } diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java index 7ab682d3143e7..204ebaf59ed35 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java @@ -7,6 +7,8 @@ */ package org.elasticsearch.persistent; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; @@ -36,6 +38,7 @@ */ public class CompletionPersistentTaskAction extends ActionType { + private static final Logger logger = LogManager.getLogger(CompletionPersistentTaskAction.class); public static final CompletionPersistentTaskAction INSTANCE = new CompletionPersistentTaskAction(); public static final String NAME = "cluster:admin/persistent/completion"; @@ -157,9 +160,11 @@ protected final void masterOperation( if (request.localAbortReason != null) { assert request.exception == null : "request has both exception " + request.exception + " and local abort reason " + request.localAbortReason; + logger.info("Persistent task unassigned due to local abort reason: [{}]", request.localAbortReason); persistentTasksClusterService.unassignPersistentTask( request.taskId, request.allocationId, + PersistentTasksCustomMetadata.Explanation.ABORTED_LOCALLY, request.localAbortReason, listener.map(PersistentTaskResponse::new) ); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index ba7b4bb51d9c7..771fbc7c062aa 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -286,12 +286,14 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) * @param taskId the id of a persistent task * @param taskAllocationId the expected allocation id of the persistent task * @param reason the reason for unassigning the task from any node + * @param reasonDetails the detailed reason string for unassigning the task from any node * @param listener the listener that will be called when task is unassigned */ public void unassignPersistentTask( final String taskId, final long taskAllocationId, - final String reason, + final PersistentTasksCustomMetadata.Explanation reason, + final String reasonDetails, final ActionListener> listener ) { submitUnbatchedTask("unassign persistent task from any node", new ClusterStateUpdateTask() { @@ -300,7 +302,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { PersistentTasksCustomMetadata.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(taskId, taskAllocationId)) { logger.trace("Unassigning task {} with allocation id {}", taskId, taskAllocationId); - return update(currentState, tasksInProgress.reassignTask(taskId, unassignedAssignment(reason))); + return update(currentState, tasksInProgress.reassignTask(taskId, unassignedAssignment(reason, reasonDetails))); } else { throw new ResourceNotFoundException("the task with id {} and allocation id {} doesn't exist", taskId, taskAllocationId); } @@ -336,7 +338,10 @@ private Assignment createAssignment( AssignmentDecision decision = enableDecider.canAssign(); if (decision.getType() == AssignmentDecision.Type.NO) { - return unassignedAssignment("persistent task [" + taskName + "] cannot be assigned [" + decision.getReason() + "]"); + return unassignedAssignment( + PersistentTasksCustomMetadata.Explanation.ASSIGNMENTS_NOT_ALLOWED, + "persistent task [" + taskName + "] cannot be assigned [" + decision.getReason() + "]" + ); } // Filter all nodes that are marked as shutting down, because we do not @@ -515,8 +520,8 @@ private static ClusterState update(ClusterState currentState, PersistentTasksCus } } - private static Assignment unassignedAssignment(String reason) { - return new Assignment(null, reason); + private static Assignment unassignedAssignment(PersistentTasksCustomMetadata.Explanation reason, String details) { + return new Assignment(null, details, reason); } /** diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java index 5fdac777b5a75..57a7befbf3846 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ObjectParser.NamedObjectParser; @@ -33,10 +34,12 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -47,7 +50,9 @@ import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.Metadata.ALL_CONTEXTS; +import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.Explanation.GENERIC_REASON; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; /** * A cluster state record that contains a list of all running persistent tasks @@ -56,7 +61,11 @@ public final class PersistentTasksCustomMetadata extends AbstractNamedDiffable> tasks; @@ -74,9 +83,14 @@ public PersistentTasksCustomMetadata(long lastAllocationId, Map ASSIGNMENT_PARSER = new ConstructingObjectParser<>( "assignment", - objects -> new Assignment((String) objects[0], (String) objects[1]) + a -> new Assignment( + (String) a[0], + (String) a[1], + a[2] == null ? new String[] { GENERIC_REASON.name() } : ((List) a[2]).toArray(String[]::new) + ) ); private static final NamedObjectParser, Void> TASK_DESCRIPTION_PARSER; @@ -103,6 +117,7 @@ public PersistentTasksCustomMetadata(long lastAllocationId, Map + * More details can be found in the {@code Assignment.explanationDetails}. + * Callers might need to interpret an assignment failure to e.g. return a HTTP status code. + * A string is hard to interpret and therefore this an {@code Explanation} enum is provided. + */ + public enum Explanation { + /** + * Persistent task assignment to this node was successful. + */ + ASSIGNMENT_SUCCESSFUL, + + /** + * Persistent task is waiting for initial assignment. This is the default when creating a new task. + */ + WAITING_FOR_INITIAL_ASSIGNMENT, + + /** + * Persistent task is awaiting reassignment after node loss. + */ + AWAITING_REASSIGNMENT, + + /** + * Persistent task assignments are not allowed due to cluster settings. + */ + ASSIGNMENTS_NOT_ALLOWED, + + /** + * Persistent task could not find appropriate nodes. + */ + NO_NODES_FOUND, + + /** + * Persistent task assignment successful but source index was removed. Task will fail during node operation. + */ + SOURCE_INDEX_REMOVED, + + /** + * Persistent task cannot be assigned during an upgrade. + */ + AWAITING_UPGRADE, + + /** + * Persistent task cannot be assigned during a feature reset. + */ + FEATURE_RESET_IN_PROGRESS, + + /** + * Persistent task was aborted locally. + */ + ABORTED_LOCALLY, + + /** + * Persistent datafeed job task state is not OPENING or OPENED. + */ + DATAFEED_JOB_STATE_NOT_OPEN, + + /** + * Persistent datafeed job task is stale. + */ + DATAFEED_JOB_STALE, + + /** + * Persistent datafeed job task index not found. + */ + DATAFEED_INDEX_NOT_FOUND, + + /** + * Persistent datafeed job task cannot start because resolving the index threw an exception. + */ + DATAFEED_RESOLVING_INDEX_THREW_EXCEPTION, + + /** + * Persistent task cannot start because indices do not have all primary shards active yet. + */ + PRIMARY_SHARDS_NOT_ACTIVE, + + /** + * Persistent task is awaiting lazy node assignment. + */ + AWAITING_LAZY_ASSIGNMENT, + + /** + * Persistent task cannot be started because job memory requirements are stale. + */ + MEMORY_REQUIREMENTS_STALE, + + /** + * Persistent task requires a node with a higher config version. + */ + CONFIG_VERSION_TOO_LOW, + + /** + * Persistent task cannot be started on a node which is not compatible with jobs of this type. + */ + NODE_NOT_COMPATIBLE, + + /** + * Persistent task cannot be started because an error occurred while detecting the load for this node. + */ + ERROR_DETECTING_LOAD, + + /** + * Persistent task cannot be started on a node that exceeds the maximum number of jobs allowed in opening state. + */ + MAX_CONCURRENT_EXECUTIONS_EXCEEDED, + + /** + * Persistent task cannot be started on a node that is full. + */ + NODE_FULL, + + /** + * Persistent task cannot be started on a node that is not providing accurate information to determine its load by memory. + */ + NODE_MEMORY_LOAD_UNKNOWN, + + /** + * Persistent task cannot be started ona node that is indicating that it has no native memory for machine learning. + */ + NO_NATIVE_MEMORY_FOR_ML, + + /** + * Persistent task cannot be started on a node that has insufficient available memory. + */ + INSUFFICIENT_MEMORY, + + /** + * Persistent task is not waiting for node assignment as estimated job size is greater than the largest possible job size. + */ + LARGEST_POSSIBLE_JOB_SIZE_EXCEEDED, + + /** + * Persistent task requires a remote connection but the node does not have the remote_cluster_client role. + */ + REMOTE_NOT_ENABLED, + + /** + * Persistent task cannot not be assigned because of a generic reason. + * This is mostly used in testing and for backwards compatibility. + */ + GENERIC_REASON, + } + /** * Private builder used in XContent parser to build task-specific portion (params and state) */ @@ -258,11 +418,27 @@ public static class Assignment { @Nullable private final String executorNode; private final String explanation; + private final Set explanationCodes = new HashSet<>(); + + public Assignment(String executorNode, PersistentTasksCustomMetadata.Explanation explanationCode) { + this(executorNode, "", explanationCode); + } + + public Assignment(String executorNode, String explanation, PersistentTasksCustomMetadata.Explanation explanationCode) { + this(executorNode, explanation, Collections.singleton(explanationCode)); + } - public Assignment(String executorNode, String explanation) { + private Assignment(String executorNode, String explanation, String[] explanationCodes) { + this(executorNode, explanation, Arrays.stream(explanationCodes).map(Explanation::valueOf).collect(Collectors.toSet())); + } + + public Assignment(String executorNode, String explanation, Set explanationCodes) { this.executorNode = executorNode; assert explanation != null; this.explanation = explanation; + assert explanationCodes != null; + assert explanationCodes.isEmpty() == false; + this.explanationCodes.addAll(explanationCodes); } @Nullable @@ -270,6 +446,14 @@ public String getExecutorNode() { return executorNode; } + public Set getExplanationCodes() { + return explanationCodes; + } + + public String getExplanationCodesAndExplanation() { + return String.join("|", explanationCodes.stream().map(Enum::name).toList()) + ", details: " + explanation; + } + public String getExplanation() { return explanation; } @@ -279,12 +463,14 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Assignment that = (Assignment) o; - return Objects.equals(executorNode, that.executorNode) && Objects.equals(explanation, that.explanation); + return Objects.equals(executorNode, that.executorNode) + && Objects.equals(explanation, that.explanation) + && Objects.equals(explanationCodes, that.explanationCodes); } @Override public int hashCode() { - return Objects.hash(executorNode, explanation); + return Objects.hash(executorNode, explanation, explanationCodes); } public boolean isAssigned() { @@ -293,11 +479,22 @@ public boolean isAssigned() { @Override public String toString() { - return "node: [" + executorNode + "], explanation: [" + explanation + "]"; + return "node: [" + + executorNode + + "], explanation: [" + + explanation + + "], explanationCodes: [" + + String.join("|", explanationCodes.stream().map(Enum::name).toList()) + + "]"; } + } - public static final Assignment INITIAL_ASSIGNMENT = new Assignment(null, "waiting for initial assignment"); + public static final Assignment INITIAL_ASSIGNMENT = new Assignment( + null, + "waiting for initial assignment", + Explanation.WAITING_FOR_INITIAL_ASSIGNMENT + ); /** * A record that represents a single running persistent task @@ -363,7 +560,15 @@ public PersistentTask(StreamInput in) throws IOException { taskName = in.readString(); params = (P) in.readNamedWriteable(PersistentTaskParams.class); state = in.readOptionalNamedWriteable(PersistentTaskState.class); - assignment = new Assignment(in.readOptionalString(), in.readString()); + if (in.getTransportVersion().before(TransportVersions.PERSISTENT_TASK_CUSTOM_METADATA_ASSIGNMENT_REASON_ENUM)) { + assignment = new Assignment(in.readOptionalString(), in.readString(), GENERIC_REASON); + } else { + assignment = new Assignment( + in.readOptionalString(), + in.readString(), + in.readCollectionAsSet(nested_in -> nested_in.readEnum(Explanation.class)) + ); + } allocationIdOnLastStatusUpdate = in.readOptionalLong(); } @@ -376,6 +581,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalNamedWriteable(state); out.writeOptionalString(assignment.executorNode); out.writeString(assignment.explanation); + if (out.getTransportVersion().onOrAfter(TransportVersions.PERSISTENT_TASK_CUSTOM_METADATA_ASSIGNMENT_REASON_ENUM)) { + out.writeCollection(assignment.explanationCodes, StreamOutput::writeEnum); + } out.writeOptionalLong(allocationIdOnLastStatusUpdate); } @@ -464,7 +672,12 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params xPa builder.startObject("assignment"); { builder.field("executor_node", assignment.executorNode); - builder.field("explanation", assignment.explanation); + if (builder.getRestApiVersion() == RestApiVersion.V_7) { + builder.field("explanation", assignment.explanation); + } else { + builder.stringListField("explanation_codes", assignment.explanationCodes.stream().map(Enum::name).toList()); + builder.field("explanation", assignment.explanation == null ? "" : assignment.explanation); + } } builder.endObject(); if (allocationIdOnLastStatusUpdate != null) { diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index 0bdd5999731dd..c4e4ee224f225 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -38,7 +38,11 @@ public String getTaskName() { return taskName; } - public static final Assignment NO_NODE_FOUND = new Assignment(null, "no appropriate nodes found for the assignment"); + public static final Assignment NO_NODE_FOUND = new Assignment( + null, + "no appropriate nodes found for the assignment", + PersistentTasksCustomMetadata.Explanation.NO_NODES_FOUND + ); /** * Returns the node id where the params has to be executed, @@ -50,7 +54,7 @@ public Assignment getAssignment(Params params, Collection candida if (discoveryNode == null) { return NO_NODE_FOUND; } else { - return new Assignment(discoveryNode.getId(), ""); + return new Assignment(discoveryNode.getId(), PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL); } } diff --git a/server/src/main/java/org/elasticsearch/persistent/decider/AssignmentDecision.java b/server/src/main/java/org/elasticsearch/persistent/decider/AssignmentDecision.java index 9bf8c023d9ec9..5c6dd6cf83c0c 100644 --- a/server/src/main/java/org/elasticsearch/persistent/decider/AssignmentDecision.java +++ b/server/src/main/java/org/elasticsearch/persistent/decider/AssignmentDecision.java @@ -18,12 +18,12 @@ */ public final class AssignmentDecision { - public static final AssignmentDecision YES = new AssignmentDecision(Type.YES, ""); + public static final AssignmentDecision YES = new AssignmentDecision(Type.YES, Reason.NO_REASON); private final Type type; - private final String reason; + private final Reason reason; - public AssignmentDecision(final Type type, final String reason) { + public AssignmentDecision(final Type type, final Reason reason) { this.type = Objects.requireNonNull(type); this.reason = Objects.requireNonNull(reason); } @@ -32,7 +32,7 @@ public Type getType() { return type; } - public String getReason() { + public Reason getReason() { return reason; } @@ -59,4 +59,9 @@ public static Type resolve(final String s) { return Type.valueOf(s.toUpperCase(Locale.ROOT)); } } + + public enum Reason { + NO_REASON, + PERSISTENT_TASK_ASSIGNMENTS_NOT_ALLOWED + } } diff --git a/server/src/main/java/org/elasticsearch/persistent/decider/EnableAssignmentDecider.java b/server/src/main/java/org/elasticsearch/persistent/decider/EnableAssignmentDecider.java index ae600dfda39a9..750d5d34e07e4 100644 --- a/server/src/main/java/org/elasticsearch/persistent/decider/EnableAssignmentDecider.java +++ b/server/src/main/java/org/elasticsearch/persistent/decider/EnableAssignmentDecider.java @@ -15,6 +15,7 @@ import static org.elasticsearch.common.settings.Setting.Property.Dynamic; import static org.elasticsearch.common.settings.Setting.Property.NodeScope; +import static org.elasticsearch.persistent.decider.AssignmentDecision.Reason.PERSISTENT_TASK_ASSIGNMENTS_NOT_ALLOWED; /** * {@link EnableAssignmentDecider} is used to allow/disallow the persistent tasks @@ -37,7 +38,6 @@ public final class EnableAssignmentDecider { Dynamic, NodeScope ); - public static final String ALLOCATION_NONE_EXPLANATION = "no persistent task assignments are allowed due to cluster settings"; private volatile Allocation enableAssignment; @@ -59,7 +59,7 @@ public void setEnableAssignment(final Allocation enableAssignment) { */ public AssignmentDecision canAssign() { if (enableAssignment == Allocation.NONE) { - return new AssignmentDecision(AssignmentDecision.Type.NO, ALLOCATION_NONE_EXPLANATION); + return new AssignmentDecision(AssignmentDecision.Type.NO, PERSISTENT_TASK_ASSIGNMENTS_NOT_ALLOWED); } return AssignmentDecision.YES; } diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java index 24d30963d18d8..00ed404d83d4c 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java +++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java @@ -106,7 +106,10 @@ public PersistentTasksCustomMetadata.Assignment getAssignment( if (discoveryNode == null) { return NO_NODE_FOUND; } else { - return new PersistentTasksCustomMetadata.Assignment(discoveryNode.getId(), ""); + return new PersistentTasksCustomMetadata.Assignment( + discoveryNode.getId(), + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index 199e72c3c3311..c526976265645 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -64,6 +64,8 @@ import static org.elasticsearch.persistent.PersistentTasksExecutor.NO_NODE_FOUND; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.elasticsearch.test.ClusterServiceUtils.setState; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -146,7 +148,18 @@ public void testReassignmentRequiredOnMetadataChanges() { boolean unassigned = randomBoolean(); PersistentTasksCustomMetadata tasks = PersistentTasksCustomMetadata.builder() - .addTask("_task_1", TestPersistentTasksExecutor.NAME, null, new Assignment(unassigned ? null : "_node", "_reason")) + .addTask( + "_task_1", + TestPersistentTasksExecutor.NAME, + null, + new Assignment( + unassigned ? null : "_node", + "_reason", + unassigned + ? PersistentTasksCustomMetadata.Explanation.GENERIC_REASON + : PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) + ) .build(); Metadata metadata = Metadata.builder() @@ -235,7 +248,8 @@ public void testNonClusterStateConditionAssignment() { for (PersistentTask task : tasksInProgress.tasks()) { assertThat(task.getExecutorNode(), nullValue()); assertThat(task.isAssigned(), equalTo(false)); - assertThat(task.getAssignment().getExplanation(), equalTo("non-cluster state condition prevents assignment")); + assertThat(task.getAssignment().getExplanation(), containsString("non-cluster state condition prevents assignment")); + assertThat(task.getAssignment().getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.GENERIC_REASON)); } assertThat(tasksInProgress.tasks().size(), equalTo(1)); @@ -247,7 +261,11 @@ public void testNonClusterStateConditionAssignment() { for (PersistentTask task : tasksInProgress.tasks()) { assertThat(task.getExecutorNode(), notNullValue()); assertThat(task.isAssigned(), equalTo(true)); - assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); + assertThat(task.getAssignment().getExplanation(), containsString("test assignment")); + assertThat( + task.getAssignment().getExplanationCodes(), + contains(PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ); } assertThat(tasksInProgress.tasks().size(), equalTo(1)); } @@ -297,20 +315,36 @@ public void testReassignTasks() { clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true) ); - assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); + assertThat(task.getAssignment().getExplanation(), containsString("test assignment")); + assertThat( + task.getAssignment().getExplanationCodes(), + contains(PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ); break; case "dont_assign_me": assertThat(task.getExecutorNode(), nullValue()); assertThat(task.isAssigned(), equalTo(false)); - assertThat(task.getAssignment().getExplanation(), equalTo("no appropriate nodes found for the assignment")); + assertThat(task.getAssignment().getExplanation(), containsString("no appropriate nodes found for the assignment")); + assertThat( + task.getAssignment().getExplanationCodes(), + contains(PersistentTasksCustomMetadata.Explanation.NO_NODES_FOUND) + ); break; case "assign_one": if (task.isAssigned()) { assignOneCount++; assertThat("more than one assign_one tasks are assigned", assignOneCount, lessThanOrEqualTo(1)); - assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); + assertThat(task.getAssignment().getExplanation(), containsString("test assignment")); + assertThat( + task.getAssignment().getExplanationCodes(), + contains(PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ); } else { - assertThat(task.getAssignment().getExplanation(), equalTo("only one task can be assigned at a time")); + assertThat(task.getAssignment().getExplanation(), containsString("only one task can be assigned at a time")); + assertThat( + task.getAssignment().getExplanationCodes(), + contains(PersistentTasksCustomMetadata.Explanation.GENERIC_REASON) + ); } break; default: @@ -334,7 +368,7 @@ public void testPersistentTasksChangedTaskAdded() { ClusterState previous = ClusterState.builder(new ClusterName("_name")).nodes(nodes).build(); PersistentTasksCustomMetadata tasks = PersistentTasksCustomMetadata.builder() - .addTask("_task_1", "test", null, new Assignment(null, "_reason")) + .addTask("_task_1", "test", null, new Assignment(null, "_reason", PersistentTasksCustomMetadata.Explanation.GENERIC_REASON)) .build(); ClusterState current = ClusterState.builder(new ClusterName("_name")) @@ -352,9 +386,24 @@ public void testPersistentTasksChangedTaskRemoved() { .build(); PersistentTasksCustomMetadata previousTasks = PersistentTasksCustomMetadata.builder() - .addTask("_task_1", "test", null, new Assignment("_node_1", "_reason")) - .addTask("_task_2", "test", null, new Assignment("_node_1", "_reason")) - .addTask("_task_3", "test", null, new Assignment("_node_2", "_reason")) + .addTask( + "_task_1", + "test", + null, + new Assignment("_node_1", "_reason", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ) + .addTask( + "_task_2", + "test", + null, + new Assignment("_node_1", "_reason", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ) + .addTask( + "_task_3", + "test", + null, + new Assignment("_node_2", "_reason", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ) .build(); ClusterState previous = ClusterState.builder(new ClusterName("_name")) @@ -363,8 +412,18 @@ public void testPersistentTasksChangedTaskRemoved() { .build(); PersistentTasksCustomMetadata currentTasks = PersistentTasksCustomMetadata.builder() - .addTask("_task_1", "test", null, new Assignment("_node_1", "_reason")) - .addTask("_task_3", "test", null, new Assignment("_node_2", "_reason")) + .addTask( + "_task_1", + "test", + null, + new Assignment("_node_1", "_reason", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ) + .addTask( + "_task_3", + "test", + null, + new Assignment("_node_2", "_reason", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ) .build(); ClusterState current = ClusterState.builder(new ClusterName("_name")) @@ -382,8 +441,8 @@ public void testPersistentTasksAssigned() { .build(); PersistentTasksCustomMetadata previousTasks = PersistentTasksCustomMetadata.builder() - .addTask("_task_1", "test", null, new Assignment("_node_1", "")) - .addTask("_task_2", "test", null, new Assignment(null, "unassigned")) + .addTask("_task_1", "test", null, new Assignment("_node_1", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL)) + .addTask("_task_2", "test", null, new Assignment(null, "unassigned", PersistentTasksCustomMetadata.Explanation.GENERIC_REASON)) .build(); ClusterState previous = ClusterState.builder(new ClusterName("_name")) @@ -392,8 +451,8 @@ public void testPersistentTasksAssigned() { .build(); PersistentTasksCustomMetadata currentTasks = PersistentTasksCustomMetadata.builder() - .addTask("_task_1", "test", null, new Assignment("_node_1", "")) - .addTask("_task_2", "test", null, new Assignment("_node_2", "")) + .addTask("_task_1", "test", null, new Assignment("_node_1", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL)) + .addTask("_task_2", "test", null, new Assignment("_node_2", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL)) .build(); ClusterState current = ClusterState.builder(new ClusterName("_name")) @@ -410,9 +469,20 @@ public void testNeedsReassignment() { .add(DiscoveryNodeUtils.create("_node_2")) .build(); - assertTrue(needsReassignment(new Assignment(null, "unassigned"), nodes)); - assertTrue(needsReassignment(new Assignment("_node_left", "assigned to a node that left"), nodes)); - assertFalse(needsReassignment(new Assignment("_node_1", "assigned"), nodes)); + assertTrue(needsReassignment(new Assignment(null, "unassigned", PersistentTasksCustomMetadata.Explanation.GENERIC_REASON), nodes)); + assertTrue( + needsReassignment( + new Assignment( + "_node_left", + "assigned to a node that left", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ), + nodes + ) + ); + assertFalse( + needsReassignment(new Assignment("_node_1", "assigned", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL), nodes) + ); } public void testPeriodicRecheck() throws Exception { @@ -448,12 +518,20 @@ public void testPeriodicRecheck() throws Exception { assertThat(task.isAssigned(), equalTo(false)); assertThat( task.getAssignment().getExplanation(), - equalTo( + containsString( shouldSimulateFailure ? "explanation: assign_based_on_non_cluster_state_condition" : "non-cluster state condition prevents assignment" ) ); + assertThat( + task.getAssignment().getExplanationCodes(), + contains( + shouldSimulateFailure + ? PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + : PersistentTasksCustomMetadata.Explanation.GENERIC_REASON + ) + ); } assertThat(tasksInProgress.tasks().size(), equalTo(1)); } @@ -469,7 +547,11 @@ public void testPeriodicRecheck() throws Exception { for (PersistentTask task : tasksInProgress.tasks()) { assertThat(task.getExecutorNode(), notNullValue()); assertThat(task.isAssigned(), equalTo(true)); - assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); + assertThat(task.getAssignment().getExplanation(), containsString("test assignment")); + assertThat( + task.getAssignment().getExplanationCodes(), + contains(PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ); } assertThat(tasksInProgress.tasks().size(), equalTo(1)); }); @@ -504,7 +586,8 @@ public void testPeriodicRecheckOffMaster() { for (PersistentTask task : tasksInProgress.tasks()) { assertThat(task.getExecutorNode(), nullValue()); assertThat(task.isAssigned(), equalTo(false)); - assertThat(task.getAssignment().getExplanation(), equalTo("non-cluster state condition prevents assignment")); + assertThat(task.getAssignment().getExplanation(), containsString("non-cluster state condition prevents assignment")); + assertThat(task.getAssignment().getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.GENERIC_REASON)); } assertThat(tasksInProgress.tasks().size(), equalTo(1)); } @@ -543,16 +626,24 @@ public void testUnassignTask() throws InterruptedException { Metadata.Builder metadata = Metadata.builder(clusterState.metadata()).putCustom(PersistentTasksCustomMetadata.TYPE, tasks.build()); clusterState = builder.metadata(metadata).nodes(nodes).build(); setState(clusterService, clusterState); - PersistentTasksClusterService service = createService((params, candidateNodes, currentState) -> new Assignment("_node_2", "test")); + PersistentTasksClusterService service = createService( + (params, candidateNodes, currentState) -> new Assignment( + "_node_2", + "test", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) + ); final var countDownLatch = new CountDownLatch(1); service.unassignPersistentTask( unassignedId, tasks.getLastAllocationId(), + PersistentTasksCustomMetadata.Explanation.GENERIC_REASON, "unassignment test", ActionTestUtils.assertNoFailureListener(task -> { assertThat(task.getAssignment().getExecutorNode(), is(nullValue())); assertThat(task.getId(), equalTo(unassignedId)); - assertThat(task.getAssignment().getExplanation(), equalTo("unassignment test")); + assertThat(task.getAssignment().getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.GENERIC_REASON)); + assertThat(task.getAssignment().getExplanation(), containsString("unassignment test")); countDownLatch.countDown(); }) ); @@ -575,11 +666,18 @@ public void testUnassignNonExistentTask() throws InterruptedException { Metadata.Builder metadata = Metadata.builder(clusterState.metadata()).putCustom(PersistentTasksCustomMetadata.TYPE, tasks.build()); clusterState = builder.metadata(metadata).nodes(nodes).build(); setState(clusterService, clusterState); - PersistentTasksClusterService service = createService((params, candidateNodes, currentState) -> new Assignment("_node_2", "test")); + PersistentTasksClusterService service = createService( + (params, candidateNodes, currentState) -> new Assignment( + "_node_2", + "test", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) + ); final var countDownLatch = new CountDownLatch(1); service.unassignPersistentTask( "missing-task", tasks.getLastAllocationId(), + PersistentTasksCustomMetadata.Explanation.GENERIC_REASON, "unassignment test", ActionListener.wrap(task -> fail(), e -> { assertThat(e, instanceOf(ResourceNotFoundException.class)); @@ -797,7 +895,11 @@ private Assignment assignOnlyOneTaskAtATime(Collection candidateN ).isEmpty()) { return randomNodeAssignment(candidateNodes); } else { - return new Assignment(null, "only one task can be assigned at a time"); + return new Assignment( + null, + "only one task can be assigned at a time", + PersistentTasksCustomMetadata.Explanation.GENERIC_REASON + ); } } @@ -805,7 +907,11 @@ private Assignment assignBasedOnNonClusterStateCondition(Collection nodes) { if (nodes.isEmpty()) { return NO_NODE_FOUND; } - return Optional.ofNullable(randomFrom(nodes)).map(node -> new Assignment(node.getId(), "test assignment")).orElse(NO_NODE_FOUND); + return Optional.ofNullable(randomFrom(nodes)) + .map(node -> new Assignment(node.getId(), "test assignment", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL)) + .orElse(NO_NODE_FOUND); } private String dumpEvent(ClusterChangedEvent event) { @@ -855,7 +963,7 @@ private ClusterState significantChange(ClusterState clusterState) { builder, Metadata.builder(clusterState.metadata()), PersistentTasksCustomMetadata.builder(tasks), - new Assignment(null, "change me"), + new Assignment(null, "change me", PersistentTasksCustomMetadata.Explanation.GENERIC_REASON), "never_assign" ); tasksOrNodesChanged = true; @@ -889,7 +997,7 @@ private PersistentTasksCustomMetadata removeTasksWithChangingAssignment(Persiste for (PersistentTask task : tasks.tasks()) { // Remove all unassigned tasks that cause changing assignments they might trigger a significant change if ("never_assign".equals(((TestParams) task.getParams()).getTestParam()) - && "change me".equals(task.getAssignment().getExplanation())) { + && task.getAssignment().getExplanation().contains("change me")) { logger.info("removed task with changing assignment {}", task.getId()); tasksBuilder.removeTask(task.getId()); changed = true; @@ -997,7 +1105,13 @@ private ClusterState.Builder addRandomTask( PersistentTasksCustomMetadata.Builder tasks, String node ) { - return addRandomTask(clusterStateBuilder, metadata, tasks, new Assignment(node, randomAlphaOfLength(10)), randomAlphaOfLength(10)); + return addRandomTask( + clusterStateBuilder, + metadata, + tasks, + new Assignment(node, randomAlphaOfLength(10), PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL), + randomAlphaOfLength(10) + ); } private ClusterState.Builder addRandomTask( @@ -1017,7 +1131,12 @@ private ClusterState.Builder addRandomTask( private String addTask(PersistentTasksCustomMetadata.Builder tasks, String param, String node) { String id = UUIDs.base64UUID(); - tasks.addTask(id, TestPersistentTasksExecutor.NAME, new TestParams(param), new Assignment(node, "explanation: " + param)); + tasks.addTask( + id, + TestPersistentTasksExecutor.NAME, + new TestParams(param), + new Assignment(node, "explanation: " + param, PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ); return id; } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetadataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetadataTests.java index 233c1f0dc4244..f829a866eb31d 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetadataTests.java @@ -313,7 +313,11 @@ public void testDisassociateDeadNodes_givenAssignedPersistentTask() { "task-id", taskName, emptyTaskParams(taskName), - new PersistentTasksCustomMetadata.Assignment("node1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); ClusterState originalState = ClusterState.builder(new ClusterName("persistent-tasks-tests")) @@ -341,13 +345,21 @@ public void testDisassociateDeadNodes() { "assigned-task", taskName, emptyTaskParams(taskName), - new PersistentTasksCustomMetadata.Assignment("node1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) .addTask( "task-on-deceased-node", taskName, emptyTaskParams(taskName), - new PersistentTasksCustomMetadata.Assignment("left-the-cluster", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "left-the-cluster", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); ClusterState originalState = ClusterState.builder(new ClusterName("persistent-tasks-tests")) @@ -396,9 +408,13 @@ private Assignment randomAssignment() { if (randomBoolean()) { return NO_NODE_FOUND; } else { - return new Assignment(null, randomAlphaOfLength(10)); + return new Assignment(null, randomAlphaOfLength(10), PersistentTasksCustomMetadata.Explanation.GENERIC_REASON); } } - return new Assignment(randomAlphaOfLength(10), randomAlphaOfLength(10)); + return new Assignment( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksDecidersTestCase.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksDecidersTestCase.java index 30e3a5e2f5218..5f9ee078766f3 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksDecidersTestCase.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksDecidersTestCase.java @@ -95,7 +95,12 @@ protected static ClusterState createClusterStateWithTasks(final int nbNodes, fin PersistentTasksCustomMetadata.Builder tasks = PersistentTasksCustomMetadata.builder(); for (int i = 0; i < nbTasks; i++) { - tasks.addTask("_task_" + i, "test", null, new PersistentTasksCustomMetadata.Assignment(null, "initialized")); + tasks.addTask( + "_task_" + i, + "test", + null, + new PersistentTasksCustomMetadata.Assignment(null, "initialized", PersistentTasksCustomMetadata.Explanation.GENERIC_REASON) + ); } Metadata metadata = Metadata.builder().putCustom(PersistentTasksCustomMetadata.TYPE, tasks.build()).build(); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 8178505470d9a..521a2897e46a4 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -126,7 +126,11 @@ public void testStartTask() { UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("other_" + i), - new Assignment("other_node_" + randomInt(nonLocalNodesCount), "test assignment on other node") + new Assignment( + "other_node_" + randomInt(nonLocalNodesCount), + "test assignment on other node", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (added == false && randomBoolean()) { added = true; @@ -134,7 +138,11 @@ public void testStartTask() { UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("this_param"), - new Assignment("this_node", "test assignment on this node") + new Assignment( + "this_node", + "test assignment on this node", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); } } @@ -238,7 +246,12 @@ public void testParamsStatusAndNodeTaskAreDelegated() throws Exception { PersistentTasksCustomMetadata.Builder tasks = PersistentTasksCustomMetadata.builder(); String taskId = UUIDs.base64UUID(); TestParams taskParams = new TestParams("other_0"); - tasks.addTask(taskId, TestPersistentTasksExecutor.NAME, taskParams, new Assignment("this_node", "test assignment on other node")); + tasks.addTask( + taskId, + TestPersistentTasksExecutor.NAME, + taskParams, + new Assignment("this_node", "test assignment on other node", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ); tasks.updateTaskState(taskId, taskState); Metadata.Builder metadata = Metadata.builder(state.metadata()); metadata.putCustom(PersistentTasksCustomMetadata.TYPE, tasks.build()); @@ -517,7 +530,7 @@ public void sendCompletionRequest( UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("this_param"), - new Assignment("this_node", "test assignment on this node") + new Assignment("this_node", "test assignment on this node", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) ); Metadata.Builder metadata = Metadata.builder(state.metadata()); @@ -541,7 +554,12 @@ private ClusterState addTask(ClusterState Metadata.builder(state.metadata()) .putCustom( PersistentTasksCustomMetadata.TYPE, - builder.addTask(UUIDs.base64UUID(), action, params, new Assignment(node, "test assignment")).build() + builder.addTask( + UUIDs.base64UUID(), + action, + params, + new Assignment(node, "test assignment", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ).build() ) ) .build(); @@ -557,7 +575,10 @@ private ClusterState reallocateTask(ClusterState state, String taskId, String no Metadata.builder(state.metadata()) .putCustom( PersistentTasksCustomMetadata.TYPE, - builder.reassignTask(taskId, new Assignment(node, "test assignment")).build() + builder.reassignTask( + taskId, + new Assignment(node, "test assignment", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) + ).build() ) ) .build(); diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index bf436b235d93b..35e46792de8ba 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -309,7 +309,11 @@ public static void setNonClusterStateCondition(boolean nonClusterStateCondition) @Override public Assignment getAssignment(TestParams params, Collection candidateNodes, ClusterState clusterState) { if (nonClusterStateCondition == false) { - return new Assignment(null, "non cluster state condition prevents assignment"); + return new Assignment( + null, + "non cluster state condition prevents assignment", + PersistentTasksCustomMetadata.Explanation.GENERIC_REASON + ); } if (params == null || params.getExecutorNodeAttr() == null) { return super.getAssignment(params, candidateNodes, clusterState); @@ -320,7 +324,11 @@ public Assignment getAssignment(TestParams params, Collection can discoveryNode -> params.getExecutorNodeAttr().equals(discoveryNode.getAttributes().get("test_attr")) ); if (executorNode != null) { - return new Assignment(executorNode.getId(), "test assignment"); + return new Assignment( + executorNode.getId(), + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ); } else { return NO_NODE_FOUND; } diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index 5b656598451a3..c554924595475 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -582,7 +582,8 @@ private static Metadata.Builder addHealthNode(Metadata.Builder metadataBuilder, PersistentTasksCustomMetadata.Builder tasks = PersistentTasksCustomMetadata.builder(); PersistentTasksCustomMetadata.Assignment assignment = new PersistentTasksCustomMetadata.Assignment( healthNode.getId(), - randomAlphaOfLength(10) + randomAlphaOfLength(10), + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL ); tasks.addTask(HealthNode.TASK_NAME, HealthNode.TASK_NAME, HealthNodeTaskParams.INSTANCE, assignment); return metadataBuilder.putCustom(PersistentTasksCustomMetadata.TYPE, tasks.build()); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java index 585bf2491bfc4..1d5639bbb3c16 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java @@ -126,7 +126,11 @@ public void validate(ShardFollowTask params, ClusterState clusterState) { } } - private static final Assignment NO_ASSIGNMENT = new Assignment(null, "no nodes found with data and remote cluster client roles"); + private static final Assignment NO_ASSIGNMENT = new Assignment( + null, + "no nodes found with data and remote cluster client roles", + PersistentTasksCustomMetadata.Explanation.NO_NODES_FOUND + ); @Override public Assignment getAssignment( @@ -142,7 +146,11 @@ public Assignment getAssignment( if (node == null) { return NO_ASSIGNMENT; } else { - return new Assignment(node.getId(), "node is the least loaded data node and remote cluster client"); + return new Assignment( + node.getId(), + "node is the least loaded data node and remote cluster client", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ); } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutorAssignmentTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutorAssignmentTests.java index 630aab4c78f43..582935dcb039f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutorAssignmentTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutorAssignmentTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksCustomMetadata.Assignment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -31,6 +32,8 @@ import java.util.function.BiConsumer; import java.util.function.Supplier; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -60,7 +63,8 @@ public void testRemoteClusterClientRoleWithoutDataRole() { private void runNoAssignmentTest(final Set roles) { runAssignmentTest(roles, 0, Set::of, (theSpecial, assignment) -> { assertFalse(assignment.isAssigned()); - assertThat(assignment.getExplanation(), equalTo("no nodes found with data and remote cluster client roles")); + assertThat(assignment.getExplanation(), containsString("no nodes found with data and remote cluster client roles")); + assertThat(assignment.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.NO_NODES_FOUND)); }); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java index 6281f656954e5..f98668fb6d4dc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java @@ -57,11 +57,13 @@ public final class MlTasks { public static final PersistentTasksCustomMetadata.Assignment AWAITING_UPGRADE = new PersistentTasksCustomMetadata.Assignment( null, - "persistent task cannot be assigned while upgrade mode is enabled." + "persistent task cannot be assigned while upgrade mode is enabled.", + PersistentTasksCustomMetadata.Explanation.AWAITING_UPGRADE ); public static final PersistentTasksCustomMetadata.Assignment RESET_IN_PROGRESS = new PersistentTasksCustomMetadata.Assignment( null, - "persistent task will not be assigned as a feature reset is in progress." + "persistent task will not be assigned as a feature reset is in progress.", + PersistentTasksCustomMetadata.Explanation.FEATURE_RESET_IN_PROGRESS ); // When a master node action is executed and there is no master node the transport will wait diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java index fafb9afa99f85..04945ca103181 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java @@ -378,7 +378,7 @@ public Response build(PersistentTasksCustomMetadata tasksInProgress, ClusterStat } return statsBuilder.setNode(node) .setDatafeedState(datafeedState) - .setAssignmentExplanation(maybeTask != null ? maybeTask.getAssignment().getExplanation() : null) + .setAssignmentExplanation(maybeTask != null ? maybeTask.getAssignment().getExplanationCodesAndExplanation() : null) .setTimingStats(timingStats) .setRunningState(datafeedRuntimeState.getRunningState(statsBuilder.datafeedId).orElse(null)) .build(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java index c11cefed137e9..b3acabd05789e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java @@ -47,7 +47,11 @@ public void testGetJobState() { MlTasks.jobTaskId("foo"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo"), - new PersistentTasksCustomMetadata.Assignment("bar", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "bar", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); assertEquals(JobState.OPENING, MlTasks.getJobState("foo", tasksBuilder.build())); @@ -71,7 +75,11 @@ public void testGetDatefeedState() { MlTasks.datafeedTaskId("foo"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("foo", 0L), - new PersistentTasksCustomMetadata.Assignment("bar", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "bar", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); // A task with no state means the datafeed is starting assertEquals(DatafeedState.STARTING, MlTasks.getDatafeedState("foo", tasksBuilder.build())); @@ -89,7 +97,11 @@ public void testGetSnapshotUpgradeState() { MlTasks.snapshotUpgradeTaskId("foo", "1"), MlTasks.JOB_SNAPSHOT_UPGRADE_TASK_NAME, new SnapshotUpgradeTaskParams("foo", "1"), - new PersistentTasksCustomMetadata.Assignment("bar", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "bar", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); // A task with no state means the datafeed is starting assertEquals(SnapshotUpgradeState.LOADING_OLD_STATE, MlTasks.getSnapshotUpgradeState("foo", "1", tasksBuilder.build())); @@ -109,7 +121,11 @@ public void testGetJobTask() { MlTasks.jobTaskId("foo"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo"), - new PersistentTasksCustomMetadata.Assignment("bar", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "bar", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); assertNotNull(MlTasks.getJobTask("foo", tasksBuilder.build())); @@ -124,7 +140,11 @@ public void testGetDatafeedTask() { MlTasks.datafeedTaskId("foo"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("foo", 0L), - new PersistentTasksCustomMetadata.Assignment("bar", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "bar", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); assertNotNull(MlTasks.getDatafeedTask("foo", tasksBuilder.build())); @@ -139,19 +159,31 @@ public void testOpenJobIds() { MlTasks.jobTaskId("foo-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.jobTaskId("bar"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("bar"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("df"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("df", 0L), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); assertThat(MlTasks.openJobIds(tasksBuilder.build()), containsInAnyOrder("foo-1", "bar")); @@ -169,19 +201,31 @@ public void testStartedDatafeedIds() { MlTasks.jobTaskId("job-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("df1"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("df1", 0L), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("df2"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("df2", 0L), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); assertThat(MlTasks.startedDatafeedIds(tasksBuilder.build()), containsInAnyOrder("df1", "df2")); @@ -197,19 +241,31 @@ public void testUnallocatedJobIds() { MlTasks.jobTaskId("job_with_assignment"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("job_with_assignment"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.jobTaskId("job_without_assignment"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("job_without_assignment"), - new PersistentTasksCustomMetadata.Assignment(null, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + null, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.jobTaskId("job_without_node"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("job_without_node"), - new PersistentTasksCustomMetadata.Assignment("dead-node", "expired node") + new PersistentTasksCustomMetadata.Assignment( + "dead-node", + "expired node", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); DiscoveryNodes nodes = DiscoveryNodes.builder() @@ -227,19 +283,31 @@ public void testUnallocatedDatafeedIds() { MlTasks.datafeedTaskId("datafeed_with_assignment"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("datafeed_with_assignment", 0L), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("datafeed_without_assignment"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("datafeed_without_assignment", 0L), - new PersistentTasksCustomMetadata.Assignment(null, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + null, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("datafeed_without_node"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("datafeed_without_node", 0L), - new PersistentTasksCustomMetadata.Assignment("dead_node", "expired node") + new PersistentTasksCustomMetadata.Assignment( + "dead_node", + "expired node", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); DiscoveryNodes nodes = DiscoveryNodes.builder() @@ -262,19 +330,31 @@ public void testDatafeedTasksOnNode() { MlTasks.datafeedTaskId("df1"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("df1", 0L), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.jobTaskId("job-2"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-2"), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("df2"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("df2", 0L), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); assertThat(MlTasks.datafeedTasksOnNode(tasksBuilder.build(), "node-2"), contains(hasProperty("id", equalTo("datafeed-df2")))); @@ -288,31 +368,51 @@ public void testJobTasksOnNode() { MlTasks.jobTaskId("job-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("df1"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("df1", 0L), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.jobTaskId("job-2"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-2"), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("df2"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("df2", 0L), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.jobTaskId("job-3"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-3"), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); assertThat( @@ -329,14 +429,22 @@ public void testNonFailedJobTasksOnNode() { MlTasks.jobTaskId("job-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.updateTaskState(MlTasks.jobTaskId("job-1"), new JobTaskState(JobState.FAILED, 1, "testing", Instant.now())); tasksBuilder.addTask( MlTasks.jobTaskId("job-2"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-2"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (randomBoolean()) { tasksBuilder.updateTaskState(MlTasks.jobTaskId("job-2"), new JobTaskState(JobState.OPENED, 2, "testing", Instant.now())); @@ -345,7 +453,11 @@ public void testNonFailedJobTasksOnNode() { MlTasks.jobTaskId("job-3"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-3"), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (randomBoolean()) { tasksBuilder.updateTaskState(MlTasks.jobTaskId("job-3"), new JobTaskState(JobState.FAILED, 3, "testing", Instant.now())); @@ -469,25 +581,41 @@ public void testFindMlProcessTasks() { MlTasks.jobTaskId("ad-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("ad-1"), - new PersistentTasksCustomMetadata.Assignment(randomAlphaOfLength(5), "test") + new PersistentTasksCustomMetadata.Assignment( + randomAlphaOfLength(5), + "test", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.dataFrameAnalyticsTaskId("dfa-1"), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams("dfa-1", MlConfigVersion.CURRENT, true), - new PersistentTasksCustomMetadata.Assignment(randomAlphaOfLength(5), "test assignment") + new PersistentTasksCustomMetadata.Assignment( + randomAlphaOfLength(5), + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.snapshotUpgradeTaskId("snapshot-upgrade-1", "some-snapshot-id"), MlTasks.JOB_SNAPSHOT_UPGRADE_TASK_NAME, new SnapshotUpgradeTaskParams("snapshot-upgrade-1", "some-snapshot-id"), - new PersistentTasksCustomMetadata.Assignment(randomAlphaOfLength(5), "test assignment") + new PersistentTasksCustomMetadata.Assignment( + randomAlphaOfLength(5), + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("datafeed-1"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("datafeed-1", "now"), - new PersistentTasksCustomMetadata.Assignment(randomAlphaOfLength(5), "test assignment") + new PersistentTasksCustomMetadata.Assignment( + randomAlphaOfLength(5), + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); PersistentTasksCustomMetadata tasks = tasksBuilder.build(); @@ -513,7 +641,11 @@ private static PersistentTasksCustomMetadata.PersistentTask createDataFrameAn MlTasks.dataFrameAnalyticsTaskId(jobId), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams(jobId, MlConfigVersion.CURRENT, false), - new PersistentTasksCustomMetadata.Assignment(nodeId, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + nodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (state != null) { builder.updateTaskState( diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java index 5e6f8b6b5b18e..e7aeeac45389d 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java @@ -139,7 +139,11 @@ public PersistentTasksCustomMetadata.Assignment getAssignment( var indexShardRouting = findShardRoutingTable(shardId, clusterState); if (indexShardRouting == null) { var node = selectLeastLoadedNode(clusterState, candidateNodes, DiscoveryNode::canContainData); - return new PersistentTasksCustomMetadata.Assignment(node.getId(), "a node to fail and stop this persistent task"); + return new PersistentTasksCustomMetadata.Assignment( + node.getId(), + "a node to fail and stop this persistent task", + PersistentTasksCustomMetadata.Explanation.SOURCE_INDEX_REMOVED + ); } final ShardRouting shardRouting = indexShardRouting.primaryShard(); @@ -153,7 +157,8 @@ public PersistentTasksCustomMetadata.Assignment getAssignment( .map( node -> new PersistentTasksCustomMetadata.Assignment( node.getId(), - "downsampling using node holding shard [" + shardId + "]" + "downsampling using node holding shard [" + shardId + "]", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL ) ) .orElse(NO_NODE_FOUND); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java index 06f6be27e9f3d..b64da81266f10 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.downsample.DownsampleShardTask; @@ -36,6 +37,8 @@ import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; import static org.elasticsearch.cluster.routing.TestShardRouting.shardRoutingBuilder; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; @@ -110,7 +113,8 @@ public void testGetAssignmentMissingIndex() { ); var result = executor.getAssignment(params, Set.of(node), clusterState); assertThat(result.getExecutorNode(), equalTo(node.getId())); - assertThat(result.getExplanation(), equalTo("a node to fail and stop this persistent task")); + assertThat(result.getExplanation(), containsString("a node to fail and stop this persistent task")); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.SOURCE_INDEX_REMOVED)); } private static DiscoveryNode newNode() { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java index b610ac60dab00..1d20e87452066 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java @@ -61,6 +61,7 @@ import static org.elasticsearch.xpack.core.ml.MlTasks.AWAITING_UPGRADE; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.equalTo; @@ -768,7 +769,7 @@ public void testSetUpgradeMode_ExistingTaskGetsUnassigned() throws Exception { assertBusy(() -> { try { GetDataFrameAnalyticsStatsAction.Response.Stats analyticsStats = getAnalyticsStats(jobId); - assertThat(analyticsStats.getAssignmentExplanation(), is(equalTo(AWAITING_UPGRADE.getExplanation()))); + assertThat(analyticsStats.getAssignmentExplanation(), containsString(AWAITING_UPGRADE.getExplanation())); assertThat(analyticsStats.getNode(), is(nullValue())); } catch (ElasticsearchException e) { logger.error(() -> "[" + jobId + "] Encountered exception while fetching analytics stats", e); @@ -783,7 +784,7 @@ public void testSetUpgradeMode_ExistingTaskGetsUnassigned() throws Exception { assertBusy(() -> { try { GetDataFrameAnalyticsStatsAction.Response.Stats analyticsStats = getAnalyticsStats(jobId); - assertThat(analyticsStats.getAssignmentExplanation(), is(not(equalTo(AWAITING_UPGRADE.getExplanation())))); + assertThat(analyticsStats.getAssignmentExplanation(), not(containsString(AWAITING_UPGRADE.getExplanation()))); } catch (ElasticsearchException e) { logger.error(() -> "[" + jobId + "] Encountered exception while fetching analytics stats", e); fail(e.getDetailedMessage()); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/SetUpgradeModeIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/SetUpgradeModeIT.java index 1233004552023..5d72d62fb3dee 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/SetUpgradeModeIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/SetUpgradeModeIT.java @@ -88,12 +88,12 @@ public void testEnableUpgradeMode() throws Exception { GetJobsStatsAction.Response.JobStats jobStats = getJobStats(jobId).get(0); assertThat(jobStats.getState(), is(equalTo(JobState.OPENED))); - assertThat(jobStats.getAssignmentExplanation(), is(equalTo(AWAITING_UPGRADE.getExplanation()))); + assertThat(jobStats.getAssignmentExplanation(), containsString(AWAITING_UPGRADE.getExplanation())); assertThat(jobStats.getNode(), is(nullValue())); GetDatafeedsStatsAction.Response.DatafeedStats datafeedStats = getDatafeedStats(datafeedId); assertThat(datafeedStats.getDatafeedState(), is(equalTo(DatafeedState.STARTED))); - assertThat(datafeedStats.getAssignmentExplanation(), is(equalTo(AWAITING_UPGRADE.getExplanation()))); + assertThat(datafeedStats.getAssignmentExplanation(), containsString(AWAITING_UPGRADE.getExplanation())); assertThat(datafeedStats.getNode(), is(nullValue())); // Disable the setting @@ -117,11 +117,11 @@ public void testEnableUpgradeMode() throws Exception { jobStats = getJobStats(jobId).get(0); assertThat(jobStats.getState(), is(equalTo(JobState.OPENED))); - assertThat(jobStats.getAssignmentExplanation(), is(not(equalTo(AWAITING_UPGRADE.getExplanation())))); + assertThat(jobStats.getAssignmentExplanation(), not(containsString(AWAITING_UPGRADE.getExplanation()))); datafeedStats = getDatafeedStats(datafeedId); assertThat(datafeedStats.getDatafeedState(), is(equalTo(DatafeedState.STARTED))); - assertThat(datafeedStats.getAssignmentExplanation(), is(not(equalTo(AWAITING_UPGRADE.getExplanation())))); + assertThat(datafeedStats.getAssignmentExplanation(), not(containsString(AWAITING_UPGRADE.getExplanation()))); } public void testJobOpenActionInUpgradeMode() { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalyticsConfigProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalyticsConfigProviderIT.java index e29cd4545846c..2073dc7a04c99 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalyticsConfigProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalyticsConfigProviderIT.java @@ -381,7 +381,11 @@ private static ClusterState clusterStateWithRunningAnalyticsTask(String analytic MlTasks.dataFrameAnalyticsTaskId(analyticsId), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams(analyticsId, MlConfigVersion.CURRENT, false), - new PersistentTasksCustomMetadata.Assignment("node", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); builder.updateTaskState( MlTasks.dataFrameAnalyticsTaskId(analyticsId), diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java index 1561520510c38..9e9060638ffe6 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java @@ -396,7 +396,11 @@ public void testExpandDatafeedsWithTaskData() throws Exception { MlTasks.datafeedTaskId("foo-1"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("foo-1", 0L), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); PersistentTasksCustomMetadata tasks = tasksBuilder.build(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java index bca437dbf676c..8ffed75d222fe 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java @@ -492,7 +492,11 @@ public void testExpandJobIdsWithTaskData() throws Exception { MlTasks.jobTaskId("foo-2"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-2"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); PersistentTasksCustomMetadata tasks = tasksBuilder.build(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java index f3774c8068489..e310ee8ae4f18 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java @@ -157,7 +157,7 @@ private void auditMlTasks( if (anomalyDetectionAuditor.includeNodeInfo()) { anomalyDetectionAuditor.warning( jobId, - "No node found to open job. Reasons [" + currentAssignment.getExplanation() + "]" + "No node found to open job. Reasons [" + currentAssignment.getExplanationCodesAndExplanation() + "]" ); } else { anomalyDetectionAuditor.warning(jobId, "Awaiting capacity to open job."); @@ -189,7 +189,7 @@ private void auditMlTasks( "No node found to start datafeed [" + datafeedParams.getDatafeedId() + "]. Reasons [" - + currentAssignment.getExplanation() + + currentAssignment.getExplanationCodesAndExplanation() + "]" ); } else { @@ -213,7 +213,7 @@ private void auditMlTasks( "[{}] No node found to start datafeed [{}]. Reasons [{}]", jobId, datafeedParams.getDatafeedId(), - currentAssignment.getExplanation() + currentAssignment.getExplanationCodesAndExplanation() ); } } @@ -231,7 +231,7 @@ private void auditMlTasks( if (anomalyDetectionAuditor.includeNodeInfo()) { dataFrameAnalyticsAuditor.warning( id, - "No node found to start analytics. Reasons [" + currentAssignment.getExplanation() + "]" + "No node found to start analytics. Reasons [" + currentAssignment.getExplanationCodesAndExplanation() + "]" ); } else { dataFrameAnalyticsAuditor.warning(id, "Awaiting capacity to start analytics."); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java index 07c0e3d7ea618..6c81e4d267803 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java @@ -375,7 +375,7 @@ private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats( String assignmentExplanation = null; if (analyticsTask != null) { node = analyticsTask.getExecutorNode() != null ? clusterState.nodes().get(analyticsTask.getExecutorNode()) : null; - assignmentExplanation = analyticsTask.getAssignment().getExplanation(); + assignmentExplanation = analyticsTask.getAssignment().getExplanationCodesAndExplanation(); } return new GetDataFrameAnalyticsStatsAction.Response.Stats( concreteAnalyticsId, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java index 5ceb015198df5..f47f44888aaa6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java @@ -96,7 +96,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A statsBuilder.setNode(state.getNodes().get(t.getExecutorNode())); } return statsBuilder.setUpgradeState(MlTasks.getSnapshotUpgradeState(t)) - .setAssignmentExplanation(t.getAssignment().getExplanation()) + .setAssignmentExplanation(t.getAssignment().getExplanationCodesAndExplanation()) .build(); }) .sorted( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java index c5061b77e2c6a..7e627e6663b96 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java @@ -153,7 +153,7 @@ protected void taskOperation( PersistentTasksCustomMetadata.PersistentTask pTask = MlTasks.getJobTask(jobId, tasks); DiscoveryNode node = state.nodes().get(pTask.getExecutorNode()); JobState jobState = MlTasks.getJobState(jobId, tasks); - String assignmentExplanation = pTask.getAssignment().getExplanation(); + String assignmentExplanation = pTask.getAssignment().getExplanationCodesAndExplanation(); TimeValue openTime = processManager.jobOpenTime(task).map(value -> TimeValue.timeValueSeconds(value.getSeconds())).orElse(null); jobResultsProvider.getForecastStats(jobId, parentTaskId, forecastStats -> { JobStats jobStats = new JobStats( @@ -219,7 +219,7 @@ void gatherStatsForClosedJobs( PersistentTasksCustomMetadata.PersistentTask pTask = MlTasks.getJobTask(jobId, tasks); String assignmentExplanation = null; if (pTask != null) { - assignmentExplanation = pTask.getAssignment().getExplanation(); + assignmentExplanation = pTask.getAssignment().getExplanationCodesAndExplanation(); } jobStats.set( slot, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java index 076573bb61b90..67d0af8db1079 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java @@ -308,10 +308,12 @@ private void unassignPersistentTasks( ); for (PersistentTask task : mlTasks) { + assert AWAITING_UPGRADE.getExplanationCodes().size() == 1; chainTaskExecutor.add( chainedTask -> persistentTasksClusterService.unassignPersistentTask( task.getId(), task.getAllocationId(), + AWAITING_UPGRADE.getExplanationCodes().iterator().next(), AWAITING_UPGRADE.getExplanation(), chainedTask ) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java index 05f3d6311404a..1222b4615b41b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -553,7 +553,7 @@ public boolean test(PersistentTasksCustomMetadata.PersistentTask persistentTa if (assignment != null && assignment.equals(PersistentTasksCustomMetadata.INITIAL_ASSIGNMENT) == false && assignment.isAssigned() == false) { - assignmentExplanation = assignment.getExplanation(); + assignmentExplanation = assignment.getExplanationCodesAndExplanation(); // Assignment failed due to primary shard check. // This is hopefully intermittent and we should allow another assignment attempt. if (assignmentExplanation.contains(PRIMARY_SHARDS_INACTIVE)) { @@ -562,7 +562,7 @@ public boolean test(PersistentTasksCustomMetadata.PersistentTask persistentTa exception = new ElasticsearchStatusException( "Could not start data frame analytics task, allocation explanation [{}]", RestStatus.TOO_MANY_REQUESTS, - assignment.getExplanation() + assignment.getExplanationCodesAndExplanation() ); return true; } @@ -803,17 +803,20 @@ private void executeTask(DataFrameAnalyticsTask task) { ); } - public static String nodeFilter(DiscoveryNode node, TaskParams params) { + public static JobNodeSelector.ExplanationAndDescription nodeFilter(DiscoveryNode node, TaskParams params) { String id = params.getId(); if (MlConfigVersion.fromNode(node).before(TaskParams.VERSION_INTRODUCED)) { - return "Not opening job [" - + id - + "] on node [" - + JobNodeSelector.nodeNameAndVersion(node) - + "], because the data frame analytics requires a node with ML config version [" - + TaskParams.VERSION_INTRODUCED - + "] or higher"; + return new JobNodeSelector.ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.CONFIG_VERSION_TOO_LOW, + "Not opening job [" + + id + + "] on node [" + + JobNodeSelector.nodeNameAndVersion(node) + + "], because the data frame analytics requires a node with ML config version [" + + TaskParams.VERSION_INTRODUCED + + "] or higher" + ); } return null; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index ab0ada00b8aaf..fcc9b809800f0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -732,7 +732,7 @@ public boolean test(PersistentTasksCustomMetadata.PersistentTask persistentTa if (assignment.equals(PersistentTasksCustomMetadata.INITIAL_ASSIGNMENT) == false && assignment.isAssigned() == false) { // Assignment has failed despite passing our "fast fail" validation exception = new ElasticsearchStatusException( - "Could not start datafeed, allocation explanation [" + assignment.getExplanation() + "]", + "Could not start datafeed, allocation explanation [" + assignment.getExplanationCodesAndExplanation() + "]", RestStatus.TOO_MANY_REQUESTS ); return true; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java index 31add7b37ac5f..a5ab1ec118cfd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java @@ -40,11 +40,13 @@ public class DatafeedNodeSelector { public static final PersistentTasksCustomMetadata.Assignment AWAITING_JOB_ASSIGNMENT = new PersistentTasksCustomMetadata.Assignment( null, - "datafeed awaiting job assignment." + "datafeed awaiting job assignment.", + PersistentTasksCustomMetadata.Explanation.WAITING_FOR_INITIAL_ASSIGNMENT ); public static final PersistentTasksCustomMetadata.Assignment AWAITING_JOB_RELOCATION = new PersistentTasksCustomMetadata.Assignment( null, - "datafeed awaiting job relocation." + "datafeed awaiting job relocation.", + PersistentTasksCustomMetadata.Explanation.AWAITING_REASSIGNMENT ); private final String datafeedId; @@ -75,7 +77,11 @@ public DatafeedNodeSelector( public void checkDatafeedTaskCanBeCreated() { if (MlMetadata.getMlMetadata(clusterState).isUpgradeMode()) { - String msg = "Unable to start datafeed [" + datafeedId + "] explanation [" + AWAITING_UPGRADE.getExplanation() + "]"; + String msg = "Unable to start datafeed [" + + datafeedId + + "] explanation [" + + AWAITING_UPGRADE.getExplanationCodesAndExplanation() + + "]"; LOGGER.debug(msg); Exception detail = new IllegalStateException(msg); throw new ElasticsearchStatusException( @@ -126,11 +132,12 @@ public PersistentTasksCustomMetadata.Assignment selectNode(Collection candidateNode.getId().equals(jobNode)) == false) { return AWAITING_JOB_RELOCATION; } - return new PersistentTasksCustomMetadata.Assignment(jobNode, ""); + return new PersistentTasksCustomMetadata.Assignment(jobNode, PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL); } LOGGER.debug(assignmentFailure.reason); - assert assignmentFailure.reason.isEmpty() == false; - return new PersistentTasksCustomMetadata.Assignment(null, assignmentFailure.reason); + assert assignmentFailure.reason != null; + assert assignmentFailure.details.isEmpty() == false; + return new PersistentTasksCustomMetadata.Assignment(null, assignmentFailure.details, assignmentFailure.reason); } @Nullable @@ -156,12 +163,14 @@ private AssignmentFailure checkAssignment() { + "] while state [" + JobState.OPENED + "] is required"; - priorityFailureCollector.add(new AssignmentFailure(reason, true)); + priorityFailureCollector.add( + new AssignmentFailure(PersistentTasksCustomMetadata.Explanation.DATAFEED_JOB_STATE_NOT_OPEN, reason, true) + ); } if (jobTaskState != null && jobTaskState.isStatusStale(jobTask)) { String reason = "cannot start datafeed [" + datafeedId + "], because the job's [" + jobId + "] state is stale"; - priorityFailureCollector.add(new AssignmentFailure(reason, true)); + priorityFailureCollector.add(new AssignmentFailure(PersistentTasksCustomMetadata.Explanation.DATAFEED_JOB_STALE, reason, true)); } return priorityFailureCollector.get(); @@ -183,6 +192,7 @@ private AssignmentFailure verifyIndicesActive() { // If we have remote indices we cannot check those. We should not fail as they may contain data. if (hasRemoteIndices == false && concreteIndices.length == 0) { return new AssignmentFailure( + PersistentTasksCustomMetadata.Explanation.DATAFEED_INDEX_NOT_FOUND, "cannot start datafeed [" + datafeedId + "] because index [" @@ -199,6 +209,7 @@ private AssignmentFailure verifyIndicesActive() { ); LOGGER.debug("[" + datafeedId + "] " + msg, e); return new AssignmentFailure( + PersistentTasksCustomMetadata.Explanation.DATAFEED_RESOLVING_INDEX_THREW_EXCEPTION, "cannot start datafeed [" + datafeedId + "] because it " + msg + " with exception [" + e.getMessage() + "]", true ); @@ -210,6 +221,7 @@ private AssignmentFailure verifyIndicesActive() { || routingTable.allPrimaryShardsActive() == false || routingTable.readyForSearch(clusterState) == false) { return new AssignmentFailure( + PersistentTasksCustomMetadata.Explanation.PRIMARY_SHARDS_NOT_ACTIVE, "cannot start datafeed [" + datafeedId + "] because index [" @@ -223,11 +235,13 @@ private AssignmentFailure verifyIndicesActive() { } private static class AssignmentFailure { - private final String reason; + private final PersistentTasksCustomMetadata.Explanation reason; + private final String details; private final boolean isCriticalForTaskCreation; - private AssignmentFailure(String reason, boolean isCriticalForTaskCreation) { + private AssignmentFailure(PersistentTasksCustomMetadata.Explanation reason, String details, boolean isCriticalForTaskCreation) { this.reason = reason; + this.details = details; this.isCriticalForTaskCreation = isCriticalForTaskCreation; } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java index a24e671d1fe25..d0702f9c04b84 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java @@ -28,6 +28,7 @@ import java.util.OptionalLong; import java.util.TreeMap; import java.util.function.Function; +import java.util.stream.Collectors; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.ml.MachineLearning.MAX_OPEN_JOBS_PER_NODE; @@ -53,7 +54,8 @@ public class JobNodeSelector { public static final PersistentTasksCustomMetadata.Assignment AWAITING_LAZY_ASSIGNMENT = new PersistentTasksCustomMetadata.Assignment( null, - "persistent task is awaiting node assignment." + "persistent task is awaiting node assignment.", + PersistentTasksCustomMetadata.Explanation.AWAITING_LAZY_ASSIGNMENT ); private static final Logger logger = LogManager.getLogger(JobNodeSelector.class); @@ -68,7 +70,7 @@ private static String createReason(String job, String node, String msg, Object.. private final ClusterState clusterState; private final Collection candidateNodes; private final MlMemoryTracker memoryTracker; - private final Function nodeFilter; + private final Function nodeFilter; private final NodeLoadDetector nodeLoadDetector; private final int maxLazyNodes; @@ -84,7 +86,7 @@ public JobNodeSelector( String taskName, MlMemoryTracker memoryTracker, int maxLazyNodes, - Function nodeFilter + Function nodeFilter ) { this.jobId = Objects.requireNonNull(jobId); this.taskName = Objects.requireNonNull(taskName); @@ -97,7 +99,10 @@ public JobNodeSelector( if (MachineLearning.isMlNode(node)) { return (nodeFilter != null) ? nodeFilter.apply(node) : null; } - return createReason(jobId, nodeNameOrId(node), "This node isn't a machine learning node."); + return new ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.NODE_NOT_COMPATIBLE, + createReason(jobId, nodeNameOrId(node), "This node isn't a machine learning node.") + ); }; } @@ -151,18 +156,22 @@ public PersistentTasksCustomMetadata.Assignment selectNode( memoryTracker.asyncRefresh(); String reason = "Not opening job [" + jobId + "] because job memory requirements are stale - refresh requested"; logger.debug(reason); - return new PersistentTasksCustomMetadata.Assignment(null, reason); + return new PersistentTasksCustomMetadata.Assignment( + null, + reason, + PersistentTasksCustomMetadata.Explanation.MEMORY_REQUIREMENTS_STALE + ); } - Map reasons = new TreeMap<>(); + Map reasons = new TreeMap<>(); long maxAvailableMemory = Long.MIN_VALUE; DiscoveryNode minLoadedNodeByMemory = null; long requiredMemoryForJob = estimatedMemoryFootprint; for (DiscoveryNode node : candidateNodes) { // First check conditions that would rule out the node regardless of what other tasks are assigned to it - String reason = nodeFilter.apply(node); + ExplanationAndDescription reason = nodeFilter.apply(node); if (reason != null) { - logger.trace(reason); + logger.trace(reason.description()); reasons.put(node.getName(), reason); continue; } @@ -174,8 +183,11 @@ public PersistentTasksCustomMetadata.Assignment selectNode( useAutoMemoryPercentage ); if (currentLoad.getError() != null) { - reason = createReason(jobId, nodeNameAndMlAttributes(node), currentLoad.getError()); - logger.trace(reason); + reason = new ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.ERROR_DETECTING_LOAD, + createReason(jobId, nodeNameAndMlAttributes(node), currentLoad.getError()) + ); + logger.trace(reason.description()); reasons.put(node.getName(), reason); continue; } @@ -184,50 +196,62 @@ public PersistentTasksCustomMetadata.Assignment selectNode( int maxNumberOfOpenJobs = currentLoad.getMaxJobs(); if (currentLoad.getNumAllocatingJobs() >= maxConcurrentJobAllocations) { - reason = createReason( - jobId, - nodeNameAndMlAttributes(node), - "Node exceeds [%s] the maximum number of jobs [%s] in opening state.", - currentLoad.getNumAllocatingJobs(), - maxConcurrentJobAllocations + reason = new ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.MAX_CONCURRENT_EXECUTIONS_EXCEEDED, + createReason( + jobId, + nodeNameAndMlAttributes(node), + "Node exceeds [%s] the maximum number of jobs [%s] in opening state.", + currentLoad.getNumAllocatingJobs(), + maxConcurrentJobAllocations + ) ); - logger.trace(reason); + logger.trace(reason.description()); reasons.put(node.getName(), reason); continue; } if (currentLoad.remainingJobs() == 0) { - reason = createReason( - jobId, - nodeNameAndMlAttributes(node), - "This node is full. Number of opened jobs and allocated native inference processes [%s], %s [%s].", - currentLoad.getNumAssignedJobsAndModels(), - MAX_OPEN_JOBS_PER_NODE.getKey(), - maxNumberOfOpenJobs + reason = new ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.NODE_FULL, + createReason( + jobId, + nodeNameAndMlAttributes(node), + "This node is full. Number of opened jobs and allocated native inference processes [%s], %s [%s].", + currentLoad.getNumAssignedJobsAndModels(), + MAX_OPEN_JOBS_PER_NODE.getKey(), + maxNumberOfOpenJobs + ) ); - logger.trace(reason); + logger.trace(reason.description()); reasons.put(node.getName(), reason); continue; } if (canAllocateByMemory == false) { - reason = createReason( - jobId, - nodeNameAndMlAttributes(node), - "This node is not providing accurate information to determine its load by memory." + reason = new ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.NODE_MEMORY_LOAD_UNKNOWN, + createReason( + jobId, + nodeNameAndMlAttributes(node), + "This node is not providing accurate information to determine its load by memory." + ) ); - logger.trace(reason); + logger.trace(reason.description()); reasons.put(node.getName(), reason); continue; } if (currentLoad.getMaxMlMemory() <= 0) { - reason = createReason( - jobId, - nodeNameAndMlAttributes(node), - "This node is indicating that it has no native memory for machine learning." + reason = new ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.NO_NATIVE_MEMORY_FOR_ML, + createReason( + jobId, + nodeNameAndMlAttributes(node), + "This node is indicating that it has no native memory for machine learning." + ) ); - logger.trace(reason); + logger.trace(reason.description()); reasons.put(node.getName(), reason); continue; } @@ -239,20 +263,23 @@ public PersistentTasksCustomMetadata.Assignment selectNode( } long availableMemory = currentLoad.getMaxMlMemory() - currentLoad.getAssignedJobMemory(); if (requiredMemoryForJob > availableMemory) { - reason = createReason( - jobId, - nodeNameAndMlAttributes(node), - "This node has insufficient available memory. Available memory for ML [%s (%s)], " - + "memory required by existing jobs [%s (%s)], " - + "estimated memory required for this job [%s (%s)].", - currentLoad.getMaxMlMemory(), - ByteSizeValue.ofBytes(currentLoad.getMaxMlMemory()).toString(), - currentLoad.getAssignedJobMemory(), - ByteSizeValue.ofBytes(currentLoad.getAssignedJobMemory()).toString(), - requiredMemoryForJob, - ByteSizeValue.ofBytes(requiredMemoryForJob).toString() + reason = new ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.INSUFFICIENT_MEMORY, + createReason( + jobId, + nodeNameAndMlAttributes(node), + "This node has insufficient available memory. Available memory for ML [%s (%s)], " + + "memory required by existing jobs [%s (%s)], " + + "estimated memory required for this job [%s (%s)].", + currentLoad.getMaxMlMemory(), + ByteSizeValue.ofBytes(currentLoad.getMaxMlMemory()).toString(), + currentLoad.getAssignedJobMemory(), + ByteSizeValue.ofBytes(currentLoad.getAssignedJobMemory()).toString(), + requiredMemoryForJob, + ByteSizeValue.ofBytes(requiredMemoryForJob).toString() + ) ); - logger.trace(reason); + logger.trace(reason.description()); reasons.put(node.getName(), reason); continue; } @@ -277,14 +304,17 @@ public PersistentTasksCustomMetadata.Assignment selectNode( PersistentTasksCustomMetadata.Assignment createAssignment( long estimatedMemoryUsage, DiscoveryNode minLoadedNode, - Collection reasons, + Collection reasons, long mostAvailableMemoryForML, long maxNodeSize ) { if (minLoadedNode == null) { - String explanation = String.join("|", reasons); - PersistentTasksCustomMetadata.Assignment currentAssignment = new PersistentTasksCustomMetadata.Assignment(null, explanation); - logger.debug("no node selected for job [{}], reasons [{}]", jobId, explanation); + PersistentTasksCustomMetadata.Assignment currentAssignment = new PersistentTasksCustomMetadata.Assignment( + null, + reasons.stream().map(ExplanationAndDescription::description).collect(Collectors.joining("|")), + reasons.stream().map(ExplanationAndDescription::explanation).collect(Collectors.toSet()) + ); + logger.debug("no node selected for job [{}], reasons [{}]", jobId, currentAssignment); if ((MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes() + estimatedMemoryUsage) > mostAvailableMemoryForML) { String message = format( "[%s] not waiting for node assignment as estimated job size [%s] is greater than largest possible job size [%s]", @@ -293,15 +323,23 @@ PersistentTasksCustomMetadata.Assignment createAssignment( mostAvailableMemoryForML ); logger.info(message); - List newReasons = new ArrayList<>(reasons); - newReasons.add(message); - explanation = String.join("|", newReasons); - return new PersistentTasksCustomMetadata.Assignment(null, explanation); + List newReasons = new ArrayList<>(reasons); + newReasons.add( + new ExplanationAndDescription(PersistentTasksCustomMetadata.Explanation.LARGEST_POSSIBLE_JOB_SIZE_EXCEEDED, message) + ); + return new PersistentTasksCustomMetadata.Assignment( + null, + newReasons.stream().map(ExplanationAndDescription::description).collect(Collectors.joining("|")), + newReasons.stream().map(ExplanationAndDescription::explanation).collect(Collectors.toSet()) + ); } return considerLazyAssignment(currentAssignment, maxNodeSize); } logger.debug("selected node [{}] for job [{}]", minLoadedNode, jobId); - return new PersistentTasksCustomMetadata.Assignment(minLoadedNode.getId(), ""); + return new PersistentTasksCustomMetadata.Assignment( + minLoadedNode.getId(), + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ); } PersistentTasksCustomMetadata.Assignment considerLazyAssignment( @@ -363,4 +401,11 @@ public static String nodeNameAndMlAttributes(DiscoveryNode node) { return builder.toString(); } + /** + * Container for returning an explanation paired with a detailed description of why an assignment is not possible. + * @param explanation High level enum explanation + * @param description Detailed text description + */ + public record ExplanationAndDescription(PersistentTasksCustomMetadata.Explanation explanation, String description) {} + } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java index 89180cba77dfd..d80778d43066f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java @@ -171,28 +171,34 @@ private static boolean nodeSupportsModelSnapshotVersion(DiscoveryNode node, Job return MlConfigVersion.getMlConfigVersionForNode(node).onOrAfter(job.getModelSnapshotMinVersion()); } - public static String nodeFilter(DiscoveryNode node, Job job) { + public static JobNodeSelector.ExplanationAndDescription nodeFilter(DiscoveryNode node, Job job) { String jobId = job.getId(); if (nodeSupportsModelSnapshotVersion(node, job) == false) { - return "Not opening job [" - + jobId - + "] on node [" - + JobNodeSelector.nodeNameAndVersion(node) - + "], because the job's model snapshot requires a node with ML config version [" - + job.getModelSnapshotMinVersion() - + "] or higher"; + return new JobNodeSelector.ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.CONFIG_VERSION_TOO_LOW, + "Not opening job [" + + jobId + + "] on node [" + + JobNodeSelector.nodeNameAndVersion(node) + + "], because the job's model snapshot requires a node with ML config version [" + + job.getModelSnapshotMinVersion() + + "] or higher" + ); } if (Job.getCompatibleJobTypes(MlConfigVersion.getMlConfigVersionForNode(node)).contains(job.getJobType()) == false) { - return "Not opening job [" - + jobId - + "] on node [" - + JobNodeSelector.nodeNameAndVersion(node) - + "], because this node does not support jobs of type [" - + job.getJobType() - + "]"; + return new JobNodeSelector.ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.NODE_NOT_COMPATIBLE, + "Not opening job [" + + jobId + + "] on node [" + + JobNodeSelector.nodeNameAndVersion(node) + + "], because this node does not support jobs of type [" + + job.getJobType() + + "]" + ); } return null; @@ -230,7 +236,7 @@ public void validate(OpenJobAction.JobParams params, ClusterState clusterState) } if (assignment.getExecutorNode() == null && assignment.equals(AWAITING_LAZY_ASSIGNMENT) == false) { - throw makeNoSuitableNodesException(logger, params.getJobId(), assignment.getExplanation()); + throw makeNoSuitableNodesException(logger, params.getJobId(), assignment.getExplanationCodesAndExplanation()); } } @@ -629,10 +635,10 @@ public static Optional checkAssignmentState( // Assignment has failed on the master node despite passing our "fast fail" validation if (assignment.equals(AWAITING_UPGRADE)) { return Optional.of(makeCurrentlyBeingUpgradedException(logger, jobId)); - } else if (assignment.getExplanation().contains("[" + EnableAssignmentDecider.ALLOCATION_NONE_EXPLANATION + "]")) { + } else if (assignment.getExplanationCodes().contains(PersistentTasksCustomMetadata.Explanation.ASSIGNMENTS_NOT_ALLOWED)) { return Optional.of(makeAssignmentsNotAllowedException(logger, jobId)); } else { - return Optional.of(makeNoSuitableNodesException(logger, jobId, assignment.getExplanation())); + return Optional.of(makeNoSuitableNodesException(logger, jobId, assignment.getExplanationCodesAndExplanation())); } } return Optional.empty(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java index 32543b45259c2..19c9950a1f107 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java @@ -235,7 +235,13 @@ public Optional checkRequiredIndices( + String.join(",", unavailableIndices) + "]"; logger.debug(reason); - return Optional.of(new PersistentTasksCustomMetadata.Assignment(null, reason)); + return Optional.of( + new PersistentTasksCustomMetadata.Assignment( + null, + reason, + PersistentTasksCustomMetadata.Explanation.PRIMARY_SHARDS_NOT_ACTIVE + ) + ); } return Optional.empty(); } @@ -246,7 +252,13 @@ public Optional checkMemoryFreshness(S if (scheduledRefresh) { String reason = "Not opening job [" + jobId + "] because job memory requirements are stale - refresh requested"; logger.debug(reason); - return Optional.of(new PersistentTasksCustomMetadata.Assignment(null, reason)); + return Optional.of( + new PersistentTasksCustomMetadata.Assignment( + null, + reason, + PersistentTasksCustomMetadata.Explanation.MEMORY_REQUIREMENTS_STALE + ) + ); } } return Optional.empty(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlLifeCycleServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlLifeCycleServiceTests.java index bdabb42ecd467..3d0b89254aadf 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlLifeCycleServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlLifeCycleServiceTests.java @@ -84,25 +84,41 @@ public void testIsNodeSafeToShutdown() { MlTasks.jobTaskId("job-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("job-1"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.datafeedTaskId("df1"), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams("df1", 0L), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.dataFrameAnalyticsTaskId("job-2"), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams("foo-2", MlConfigVersion.CURRENT, true), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( MlTasks.snapshotUpgradeTaskId("job-3", "snapshot-3"), MlTasks.JOB_SNAPSHOT_UPGRADE_TASK_NAME, new SnapshotUpgradeTaskParams("job-3", "snapshot-3"), - new PersistentTasksCustomMetadata.Assignment("node-3", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-3", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); Metadata metadata = Metadata.builder().putCustom(PersistentTasksCustomMetadata.TYPE, tasksBuilder.build()).build(); @@ -145,14 +161,22 @@ public void testIsNodeSafeToShutdownGivenFailedTasks() { MlTasks.jobTaskId("job-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("job-1"), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.updateTaskState(MlTasks.jobTaskId("job-1"), new JobTaskState(JobState.FAILED, 1, "testing", Instant.now())); tasksBuilder.addTask( MlTasks.dataFrameAnalyticsTaskId("job-2"), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams("foo-2", MlConfigVersion.CURRENT, true), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.updateTaskState( MlTasks.dataFrameAnalyticsTaskId("job-2"), @@ -162,7 +186,11 @@ public void testIsNodeSafeToShutdownGivenFailedTasks() { MlTasks.snapshotUpgradeTaskId("job-3", "snapshot-3"), MlTasks.JOB_SNAPSHOT_UPGRADE_TASK_NAME, new SnapshotUpgradeTaskParams("job-3", "snapshot-3"), - new PersistentTasksCustomMetadata.Assignment("node-3", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-3", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.updateTaskState( MlTasks.snapshotUpgradeTaskId("job-3", "snapshot-3"), diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetricsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetricsTests.java index 5fb1381b881ea..41f149d74affb 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetricsTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetricsTests.java @@ -174,7 +174,13 @@ public static void addDatafeedTask( MlTasks.datafeedTaskId(datafeedId), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams(datafeedId, System.currentTimeMillis()), - nodeId == null ? AWAITING_LAZY_ASSIGNMENT : new PersistentTasksCustomMetadata.Assignment(nodeId, "test assignment") + nodeId == null + ? AWAITING_LAZY_ASSIGNMENT + : new PersistentTasksCustomMetadata.Assignment( + nodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (datafeedState != null) { builder.updateTaskState(MlTasks.datafeedTaskId(datafeedId), datafeedState); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index 67da449af850f..1cce5fdaf95d0 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -332,7 +332,7 @@ public static void addTask( MlTasks.datafeedTaskId(datafeedId), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams(datafeedId, startTime), - new Assignment(nodeId, "test assignment") + new Assignment(nodeId, "test assignment", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) ); tasks.updateTaskState(MlTasks.datafeedTaskId(datafeedId), state); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java index 64d1414134f38..7d49339bb6c4e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksCustomMetadata.Assignment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -42,7 +43,9 @@ import java.util.Set; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -64,7 +67,7 @@ public void testGetAssignment_UpgradeModeIsEnabled() { Assignment assignment = executor.getAssignment(params, clusterState.nodes().getAllNodes(), clusterState); assertThat(assignment.getExecutorNode(), is(nullValue())); - assertThat(assignment.getExplanation(), is(equalTo("persistent task cannot be assigned while upgrade mode is enabled."))); + assertThat(assignment.getExplanation(), containsString("persistent task cannot be assigned while upgrade mode is enabled.")); } // Cannot assign the node because there are no existing nodes in the cluster state @@ -77,6 +80,7 @@ public void testGetAssignment_NoNodes() { Assignment assignment = executor.getAssignment(params, clusterState.nodes().getAllNodes(), clusterState); assertThat(assignment.getExecutorNode(), is(nullValue())); + assertThat(assignment.getExplanationCodes(), empty()); assertThat(assignment.getExplanation(), is(emptyString())); } @@ -96,6 +100,7 @@ public void testGetAssignment_NoMlNodes() { Assignment assignment = executor.getAssignment(params, clusterState.nodes().getAllNodes(), clusterState); assertThat(assignment.getExecutorNode(), is(nullValue())); + assertThat(assignment.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.NODE_NOT_COMPATIBLE)); assertThat( assignment.getExplanation(), allOf( @@ -118,6 +123,7 @@ public void testGetAssignment_MlNodeIsNewerThanTheMlJobButTheAssignmentSuceeds() Assignment assignment = executor.getAssignment(params, clusterState.nodes().getAllNodes(), clusterState); assertThat(assignment.getExecutorNode(), is(equalTo("_node_id0"))); + assertThat(assignment.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL)); assertThat(assignment.getExplanation(), is(emptyString())); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsActionTests.java index d1d3338ce14ba..19a70e5306466 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsActionTests.java @@ -79,7 +79,11 @@ private static void addAnalyticsTask( MlTasks.dataFrameAnalyticsTaskId(analyticsId), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams(analyticsId, MlConfigVersion.CURRENT, allowLazyStart), - new PersistentTasksCustomMetadata.Assignment(nodeId, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + nodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (state != null) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java index c1dd5ce07e569..054f7cbecd45a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java @@ -100,7 +100,11 @@ public static void addTask( MlTasks.datafeedTaskId(datafeedId), MlTasks.DATAFEED_TASK_NAME, new StartDatafeedAction.DatafeedParams(datafeedId, startTime), - new PersistentTasksCustomMetadata.Assignment(nodeId, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + nodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); taskBuilder.updateTaskState(MlTasks.datafeedTaskId(datafeedId), state); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java index 970044c188849..b08d6a7f41c88 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java @@ -1347,7 +1347,11 @@ private static ClusterState clusterState( MlTasks.datafeedTaskId(jobId + "-datafeed"), MlTasks.DATAFEED_TASK_NAME, dfParams, - new PersistentTasksCustomMetadata.Assignment(nodeAssignment, "test") + new PersistentTasksCustomMetadata.Assignment( + nodeAssignment, + "test", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); } for (String analyticsId : analyticsTasks) { @@ -1409,7 +1413,13 @@ public static void addAnalyticsTask( MlTasks.dataFrameAnalyticsTaskId(jobId), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams(jobId, MlConfigVersion.CURRENT, true), - nodeId == null ? AWAITING_LAZY_ASSIGNMENT : new PersistentTasksCustomMetadata.Assignment(nodeId, "test assignment") + nodeId == null + ? AWAITING_LAZY_ASSIGNMENT + : new PersistentTasksCustomMetadata.Assignment( + nodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (jobState != null) { builder.updateTaskState( @@ -1424,7 +1434,13 @@ public static void addJobTask(String jobId, String nodeId, JobState jobState, Pe MlTasks.jobTaskId(jobId), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams(jobId), - nodeId == null ? AWAITING_LAZY_ASSIGNMENT : new PersistentTasksCustomMetadata.Assignment(nodeId, "test assignment") + nodeId == null + ? AWAITING_LAZY_ASSIGNMENT + : new PersistentTasksCustomMetadata.Assignment( + nodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (jobState != null) { builder.updateTaskState( diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index 4bb612921876e..bcddc0a0c2b54 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -57,6 +57,7 @@ import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeed; import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -175,10 +176,11 @@ public void testNoJobTask() { assertNull(result.getExecutorNode()); assertThat( result.getExplanation(), - equalTo( + containsString( "cannot start datafeed [datafeed_id], because the job's [job_id] state is " + "[closed] while state [opened] is required" ) ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.DATAFEED_JOB_STATE_NOT_OPEN)); ElasticsearchException e = expectThrows( ElasticsearchException.class, @@ -224,6 +226,7 @@ public void testSelectNode_GivenJobFailedOrClosed() { "cannot start datafeed [datafeed_id], because the job's [job_id] state is [" + jobState + "] while state [opened] is required", result.getExplanation() ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.DATAFEED_JOB_STATE_NOT_OPEN)); ElasticsearchException e = expectThrows( ElasticsearchException.class, @@ -273,8 +276,9 @@ public void testShardUnassigned() { assertNull(result.getExecutorNode()); assertThat( result.getExplanation(), - equalTo("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.") + containsString("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.") ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.PRIMARY_SHARDS_NOT_ACTIVE)); new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), SearchRequest.DEFAULT_INDICES_OPTIONS) .checkDatafeedTaskCanBeCreated(); @@ -307,8 +311,9 @@ public void testShardNotAllActive() { assertNull(result.getExecutorNode()); assertThat( result.getExplanation(), - equalTo("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.") + containsString("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.") ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.PRIMARY_SHARDS_NOT_ACTIVE)); new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), SearchRequest.DEFAULT_INDICES_OPTIONS) .checkDatafeedTaskCanBeCreated(); @@ -337,14 +342,14 @@ public void testIndexDoesntExist() { result.getExplanation(), anyOf( // TODO remove this first option and only allow the second once the failure store functionality is permanently switched on - equalTo( + containsString( "cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and " + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, " + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, " + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true]] " + "with exception [no such index [not_foo]]" ), - equalTo( + containsString( "cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and " + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, " + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, " @@ -353,6 +358,10 @@ public void testIndexDoesntExist() { ) ) ); + assertThat( + result.getExplanationCodes(), + contains(PersistentTasksCustomMetadata.Explanation.DATAFEED_RESOLVING_INDEX_THREW_EXCEPTION) + ); ElasticsearchException e = expectThrows( ElasticsearchException.class, @@ -482,6 +491,7 @@ public void testSelectNode_jobTaskStale() { ).selectNode(candidateNodes); assertNull(result.getExecutorNode()); assertEquals("cannot start datafeed [datafeed_id], because the job's [job_id] state is stale", result.getExplanation()); + assertEquals(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.DATAFEED_JOB_STALE)); ElasticsearchException e = expectThrows( ElasticsearchException.class, @@ -656,6 +666,7 @@ public void testSelectNode_GivenJobIsOpenedAndNodeIsShuttingDown() { ).selectNode(makeCandidateNodes("other_node_id")); assertNull(result.getExecutorNode()); assertEquals("datafeed awaiting job relocation.", result.getExplanation()); + assertEquals(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.AWAITING_REASSIGNMENT)); // This is different to the pattern of the other tests - we allow the datafeed task to be // created even though it cannot be assigned. The reason is that it would be perverse for diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java index 1dc44582492aa..e66ce064e25bb 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java @@ -1330,7 +1330,11 @@ public void testDetectReasonToRebalanceModels_GivenMultipleMlJobsStopped() { MlTasks.dataFrameAnalyticsTaskId("dfa-1"), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams("dfa-1", MlConfigVersion.CURRENT, true), - new PersistentTasksCustomMetadata.Assignment(mlNodeId, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + mlNodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); PersistentTasksCustomMetadata.Builder currentTasksBuilder = PersistentTasksCustomMetadata.builder(); @@ -1401,7 +1405,11 @@ public void testDetectReasonToRebalanceModels_GivenMlJobsStarted() { MlTasks.dataFrameAnalyticsTaskId("dfa-1"), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams("dfa-1", MlConfigVersion.CURRENT, true), - new PersistentTasksCustomMetadata.Assignment(mlNodeId, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + mlNodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); PersistentTasksCustomMetadata.Builder currentTasksBuilder = PersistentTasksCustomMetadata.builder(); @@ -1421,7 +1429,11 @@ public void testDetectReasonToRebalanceModels_GivenMlJobsStarted() { MlTasks.dataFrameAnalyticsTaskId("dfa-1"), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams("dfa-1", MlConfigVersion.CURRENT, true), - new PersistentTasksCustomMetadata.Assignment(mlNodeId, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + mlNodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); ClusterState previousState = ClusterState.builder(new ClusterName("test_cluster")) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java index 112a8c80b0483..f829f9d08c29a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java @@ -51,6 +51,7 @@ import static org.elasticsearch.xpack.ml.job.task.OpenJobPersistentTasksExecutor.nodeFilter; import static org.elasticsearch.xpack.ml.job.task.OpenJobPersistentTasksExecutorTests.jobWithRules; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.anyString; @@ -152,6 +153,7 @@ public void testSelectLeastLoadedMlNodeForAnomalyDetectorJob_maxCapacityCountLim false ); assertNull(result.getExecutorNode()); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.NODE_FULL)); assertThat( result.getExplanation(), containsString( @@ -210,6 +212,7 @@ public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_maxCapacityCount + "]" ) ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.NODE_FULL)); } public void testSelectLeastLoadedMlNodeForAnomalyDetectorJob_maxCapacityMemoryLimiting() { @@ -273,6 +276,7 @@ public void testSelectLeastLoadedMlNodeForAnomalyDetectorJob_maxCapacityMemoryLi + ")]" ) ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.INSUFFICIENT_MEMORY)); } public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_givenTaskHasNullState() { @@ -364,6 +368,7 @@ public void testSelectLeastLoadedMlNodeForAnomalyDetectorJob_firstJobTooBigMemor + ")]" ) ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.INSUFFICIENT_MEMORY)); } public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_maxCapacityMemoryLimiting() { @@ -427,6 +432,7 @@ public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_maxCapacityMemor + ")]" ) ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.INSUFFICIENT_MEMORY)); } public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_firstJobTooBigMemoryLimiting() { @@ -481,6 +487,7 @@ public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_firstJobTooBigMe + ")]" ) ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.INSUFFICIENT_MEMORY)); } public void testSelectLeastLoadedMlNode_noMlNodes() { @@ -528,6 +535,7 @@ public void testSelectLeastLoadedMlNode_noMlNodes() { ); PersistentTasksCustomMetadata.Assignment result = jobNodeSelector.selectNode(20, 2, 30, MAX_JOB_BYTES, false); assertTrue(result.getExplanation().contains("node isn't a machine learning node")); + assertTrue(result.getExplanationCodes().contains(PersistentTasksCustomMetadata.Explanation.NODE_NOT_COMPATIBLE)); assertNull(result.getExecutorNode()); } @@ -620,11 +628,16 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { result = jobNodeSelector.selectNode(10, 2, 30, MAX_JOB_BYTES, false); assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("Node exceeds [2] the maximum number of jobs [2] in opening state")); + assertTrue(result.getExplanationCodes().contains(PersistentTasksCustomMetadata.Explanation.MAX_CONCURRENT_EXECUTIONS_EXCEEDED)); tasksBuilder = PersistentTasksCustomMetadata.builder(tasks); tasksBuilder.reassignTask( MlTasks.jobTaskId(job6.getId()), - new PersistentTasksCustomMetadata.Assignment("_node_id3", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "_node_id3", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasks = tasksBuilder.build(); @@ -643,6 +656,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { result = jobNodeSelector.selectNode(10, 2, 30, MAX_JOB_BYTES, false); assertNull("no node selected, because stale task", result.getExecutorNode()); assertTrue(result.getExplanation().contains("Node exceeds [2] the maximum number of jobs [2] in opening state")); + assertTrue(result.getExplanationCodes().contains(PersistentTasksCustomMetadata.Explanation.MAX_CONCURRENT_EXECUTIONS_EXCEEDED)); tasksBuilder = PersistentTasksCustomMetadata.builder(tasks); tasksBuilder.updateTaskState(MlTasks.jobTaskId(job6.getId()), null); @@ -663,6 +677,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { result = jobNodeSelector.selectNode(10, 2, 30, MAX_JOB_BYTES, false); assertNull("no node selected, because null state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("Node exceeds [2] the maximum number of jobs [2] in opening state")); + assertTrue(result.getExplanationCodes().contains(PersistentTasksCustomMetadata.Explanation.MAX_CONCURRENT_EXECUTIONS_EXCEEDED)); } public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() { @@ -709,7 +724,11 @@ public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() // This will make the assignment stale for job_id1 tasksBuilder.reassignTask( MlTasks.jobTaskId("job_id1"), - new PersistentTasksCustomMetadata.Assignment("_node_id1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "_node_id1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); OpenJobPersistentTasksExecutorTests.addJobTask("job_id2", "_node_id1", null, tasksBuilder); OpenJobPersistentTasksExecutorTests.addJobTask("job_id3", "_node_id2", null, tasksBuilder); @@ -760,6 +779,7 @@ public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() result = jobNodeSelector.selectNode(10, 2, 30, MAX_JOB_BYTES, false); assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("Node exceeds [2] the maximum number of jobs [2] in opening state")); + assertTrue(result.getExplanationCodes().contains(PersistentTasksCustomMetadata.Explanation.MAX_CONCURRENT_EXECUTIONS_EXCEEDED)); } public void testSelectLeastLoadedMlNode_noCompatibleJobTypeNodes() { @@ -819,6 +839,7 @@ public void testSelectLeastLoadedMlNode_noCompatibleJobTypeNodes() { ); PersistentTasksCustomMetadata.Assignment result = jobNodeSelector.selectNode(10, 2, 30, MAX_JOB_BYTES, false); assertThat(result.getExplanation(), containsString("node does not support jobs of type [incompatible_type]")); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.NODE_NOT_COMPATIBLE)); assertNull(result.getExecutorNode()); } @@ -880,7 +901,7 @@ public void testSelectLeastLoadedMlNode_reasonsAreInDeterministicOrder() { PersistentTasksCustomMetadata.Assignment result = jobNodeSelector.selectNode(10, 2, 30, MAX_JOB_BYTES, false); assertThat( result.getExplanation(), - equalTo( + containsString( "Not opening job [incompatible_type_job] on node [{_node_name1}{ML config version=" + MlConfigVersion.CURRENT + "}], " @@ -891,6 +912,7 @@ public void testSelectLeastLoadedMlNode_reasonsAreInDeterministicOrder() { + "because this node does not support jobs of type [incompatible_type]" ) ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.NODE_NOT_COMPATIBLE)); assertNull(result.getExecutorNode()); } @@ -960,6 +982,7 @@ public void testSelectLeastLoadedMlNode_noNodesMatchingModelSnapshotMinVersion() result.getExplanation(), containsString("job's model snapshot requires a node with ML config version [7.3.0] or higher") ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.CONFIG_VERSION_TOO_LOW)); assertNull(result.getExecutorNode()); } @@ -1110,10 +1133,10 @@ public void testConsiderLazyAssignmentWithNoLazyNodes() { node -> nodeFilter(node, job) ); PersistentTasksCustomMetadata.Assignment result = jobNodeSelector.considerLazyAssignment( - new PersistentTasksCustomMetadata.Assignment(null, "foo"), + new PersistentTasksCustomMetadata.Assignment(null, "foo", PersistentTasksCustomMetadata.Explanation.GENERIC_REASON), ByteSizeValue.ofGb(1).getBytes() ); - assertEquals("foo", result.getExplanation()); + assertThat(result.getExplanation(), containsString("foo")); assertNull(result.getExecutorNode()); } @@ -1153,10 +1176,10 @@ public void testConsiderLazyAssignmentWithLazyNodes() { node -> nodeFilter(node, job) ); PersistentTasksCustomMetadata.Assignment result = jobNodeSelector.considerLazyAssignment( - new PersistentTasksCustomMetadata.Assignment(null, "foo"), + new PersistentTasksCustomMetadata.Assignment(null, "foo", PersistentTasksCustomMetadata.Explanation.GENERIC_REASON), ByteSizeValue.ofGb(1).getBytes() ); - assertEquals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT.getExplanation(), result.getExplanation()); + assertEquals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT.getExplanationCodes(), result.getExplanationCodes()); assertNull(result.getExecutorNode()); } @@ -1206,10 +1229,10 @@ public void testConsiderLazyAssignmentWithFilledLazyNodesAndVerticalScale() { node -> nodeFilter(node, job) ); PersistentTasksCustomMetadata.Assignment result = jobNodeSelector.considerLazyAssignment( - new PersistentTasksCustomMetadata.Assignment(null, "foo"), + new PersistentTasksCustomMetadata.Assignment(null, "foo", PersistentTasksCustomMetadata.Explanation.GENERIC_REASON), ByteSizeValue.ofGb(64).getBytes() ); - assertEquals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT.getExplanation(), result.getExplanation()); + assertEquals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT.getExplanationCodes(), result.getExplanationCodes()); assertNull(result.getExecutorNode()); } @@ -1257,6 +1280,7 @@ public void testMaximumPossibleNodeMemoryTooSmall() { + "[31458280] is greater than largest possible job size [3]" ) ); + assertThat(result.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.LARGEST_POSSIBLE_JOB_SIZE_EXCEEDED)); } public void testPerceivedCapacityAndMaxFreeMemory() { @@ -1425,7 +1449,11 @@ static void addDataFrameAnalyticsJobTask( MlTasks.dataFrameAnalyticsTaskId(id), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams(id, MlConfigVersion.CURRENT, allowLazyStart), - new PersistentTasksCustomMetadata.Assignment(nodeId, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + nodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (state != null) { builder.updateTaskState( diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradePredicateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradePredicateTests.java index 2e029ed9c58c1..b9fc7f8d6df79 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradePredicateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradePredicateTests.java @@ -27,7 +27,7 @@ public void testWhenWaitForCompletionIsTrue() { MlTasks.JOB_SNAPSHOT_UPGRADE_TASK_NAME, new SnapshotUpgradeTaskParams("job", "snapshot"), 1, - new PersistentTasksCustomMetadata.Assignment("test-node", "") + new PersistentTasksCustomMetadata.Assignment("test-node", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) ); { SnapshotUpgradePredicate snapshotUpgradePredicate = new SnapshotUpgradePredicate(true, logger); @@ -65,7 +65,7 @@ public void testWhenWaitForCompletionIsFalse() { MlTasks.JOB_SNAPSHOT_UPGRADE_TASK_NAME, new SnapshotUpgradeTaskParams("job", "snapshot"), 1, - new PersistentTasksCustomMetadata.Assignment("test-node", "") + new PersistentTasksCustomMetadata.Assignment("test-node", PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL) ); { SnapshotUpgradePredicate snapshotUpgradePredicate = new SnapshotUpgradePredicate(false, logger); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java index 0440a66bdbcaa..a2aef9b57bb9c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java @@ -167,11 +167,17 @@ public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() { OpenJobAction.JobParams params = new OpenJobAction.JobParams("unavailable_index_with_lazy_node"); params.setJob(mock(Job.class)); + PersistentTasksCustomMetadata.Assignment assignment = executor.getAssignment( + params, + csBuilder.nodes().getAllNodes(), + csBuilder.build() + ); assertEquals( "Not opening [unavailable_index_with_lazy_node], " + "because not all primary shards are active for the following indices [.ml-state]", - executor.getAssignment(params, csBuilder.nodes().getAllNodes(), csBuilder.build()).getExplanation() + assignment.getExplanation() ); + assertTrue(assignment.getExplanationCodes().contains(PersistentTasksCustomMetadata.Explanation.PRIMARY_SHARDS_NOT_ACTIVE)); } public void testGetAssignment_GivenLazyJobAndNoGlobalLazyNodes() { @@ -196,7 +202,7 @@ public void testGetAssignment_GivenLazyJobAndNoGlobalLazyNodes() { ); assertNotNull(assignment); assertNull(assignment.getExecutorNode()); - assertEquals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT.getExplanation(), assignment.getExplanation()); + assertEquals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT.getExplanationCodes(), assignment.getExplanationCodes()); } public void testGetAssignment_GivenResetInProgress() { @@ -217,7 +223,7 @@ public void testGetAssignment_GivenResetInProgress() { ); assertNotNull(assignment); assertNull(assignment.getExecutorNode()); - assertEquals(MlTasks.RESET_IN_PROGRESS.getExplanation(), assignment.getExplanation()); + assertEquals(MlTasks.RESET_IN_PROGRESS.getExplanationCodes(), assignment.getExplanationCodes()); } public static void addJobTask(String jobId, String nodeId, JobState jobState, PersistentTasksCustomMetadata.Builder builder) { @@ -235,7 +241,11 @@ public static void addJobTask( MlTasks.jobTaskId(jobId), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams(jobId), - new PersistentTasksCustomMetadata.Assignment(nodeId, "test assignment") + new PersistentTasksCustomMetadata.Assignment( + nodeId, + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); if (jobState != null) { builder.updateTaskState( diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java index 8b63b76cdf248..847c348c3546a 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java @@ -331,7 +331,11 @@ public void updatePersistentTaskState( RollupField.TASK_NAME, job, 1, - new PersistentTasksCustomMetadata.Assignment("foo", "foo") + new PersistentTasksCustomMetadata.Assignment( + "foo", + "foo", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) ); counter.incrementAndGet(); @@ -433,7 +437,11 @@ public void updatePersistentTaskState( RollupField.TASK_NAME, job, 1, - new PersistentTasksCustomMetadata.Assignment("foo", "foo") + new PersistentTasksCustomMetadata.Assignment( + "foo", + "foo", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) ); } @@ -492,7 +500,11 @@ public void updatePersistentTaskState( RollupField.TASK_NAME, job, 1, - new PersistentTasksCustomMetadata.Assignment("foo", "foo") + new PersistentTasksCustomMetadata.Assignment( + "foo", + "foo", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) ); } @@ -554,7 +566,11 @@ public void updatePersistentTaskState( RollupField.TASK_NAME, job, 1, - new PersistentTasksCustomMetadata.Assignment("foo", "foo") + new PersistentTasksCustomMetadata.Assignment( + "foo", + "foo", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) ); } @@ -650,7 +666,11 @@ public void updatePersistentTaskState( RollupField.TASK_NAME, job, 1, - new PersistentTasksCustomMetadata.Assignment("foo", "foo") + new PersistentTasksCustomMetadata.Assignment( + "foo", + "foo", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) ); } else if (counterValue == 1) { @@ -759,7 +779,11 @@ public void updatePersistentTaskState( RollupField.TASK_NAME, job, 1, - new PersistentTasksCustomMetadata.Assignment("foo", "foo") + new PersistentTasksCustomMetadata.Assignment( + "foo", + "foo", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) ); } else if (counterValue == 1) { @@ -870,7 +894,11 @@ public void updatePersistentTaskState( RollupField.TASK_NAME, job, 1, - new PersistentTasksCustomMetadata.Assignment("foo", "foo") + new PersistentTasksCustomMetadata.Assignment( + "foo", + "foo", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) ); } else if (counterValue == 1) { @@ -992,7 +1020,11 @@ public void updatePersistentTaskState( RollupField.TASK_NAME, job, 1, - new PersistentTasksCustomMetadata.Assignment("foo", "foo") + new PersistentTasksCustomMetadata.Assignment( + "foo", + "foo", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) ); counter.incrementAndGet(); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java index 302db8816f4bf..7ec97843d9b49 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java @@ -396,7 +396,7 @@ private void addCheckpointingInfoForTransformsWithoutTasks( new TransformStats( stat.getId(), TransformStats.State.WAITING, - assignment.getExplanation(), + assignment.getExplanationCodesAndExplanation(), null, stat.getTransformStats(), checkpointingInfo, diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java index 23212636dc33c..cf94e0ad581bb 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java @@ -374,7 +374,7 @@ public boolean test(PersistentTasksCustomMetadata.PersistentTask persistentTa // For some reason, the task is not assigned to a node, but is no longer in the `INITIAL_ASSIGNMENT` state // Consider this a failure. exception = new ElasticsearchStatusException( - "Could not start transform, allocation explanation [" + assignment.getExplanation() + "]", + "Could not start transform, allocation explanation [" + assignment.getExplanationCodesAndExplanation() + "]", RestStatus.TOO_MANY_REQUESTS ); return true; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformHealthChecker.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformHealthChecker.java index 24c5d45a38f75..1e86e1fe80477 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformHealthChecker.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformHealthChecker.java @@ -63,7 +63,7 @@ public static TransformHealth checkUnassignedTransform( ) { final Assignment assignment = TransformNodes.getAssignment(transformId, clusterState); final List issues = new ArrayList<>(2); - issues.add(IssueType.ASSIGNMENT_FAILED.newIssue(assignment.getExplanation(), 1, null)); + issues.add(IssueType.ASSIGNMENT_FAILED.newIssue(assignment.getExplanationCodesAndExplanation(), 1, null)); if (AuthorizationState.isNullOrGreen(authState) == false) { issues.add(IssueType.PRIVILEGES_CHECK_FAILED.newIssue(authState.getLastAuthError(), 1, null)); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformNodes.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformNodes.java index 56e5fd5900cfb..a72fc13a66a80 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformNodes.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformNodes.java @@ -180,7 +180,7 @@ public static (listener, reader, TransportResponseHandler.TRANSPORT_WORKER) ); } else { - Map explain = new TreeMap<>(); + Map explain = new TreeMap<>(); for (DiscoveryNode node : nodes) { nodeCanRunThisTransform(node, null, requiresRemote, explain); } @@ -188,7 +188,10 @@ public static e.getKey() + ":" + e.getValue()).collect(Collectors.joining("|")) + explain.entrySet() + .stream() + .map(e -> e.getKey() + ":" + e.getValue().explanation() + " - " + e.getValue().description()) + .collect(Collectors.joining("|")) ) ); } @@ -214,17 +217,20 @@ public static boolean nodeCanRunThisTransform( DiscoveryNode node, TransformConfigVersion minRequiredVersion, boolean requiresRemote, - Map explain + Map explain ) { // version of the transform run on a node that has at least the same version if (minRequiredVersion != null && TransformConfigVersion.fromNode(node).onOrAfter(minRequiredVersion) == false) { if (explain != null) { explain.put( node.getId(), - "node supports transform config version: " - + TransformConfigVersion.fromNode(node) - + " but transform requires at least " - + minRequiredVersion + new ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.CONFIG_VERSION_TOO_LOW, + "node supports transform config version: " + + TransformConfigVersion.fromNode(node) + + " but transform requires at least " + + minRequiredVersion + ) ); } return false; @@ -233,7 +239,10 @@ public static boolean nodeCanRunThisTransform( // transform enabled? if (node.getRoles().contains(DiscoveryNodeRole.TRANSFORM_ROLE) == false) { if (explain != null) { - explain.put(node.getId(), "not a transform node"); + explain.put( + node.getId(), + new ExplanationAndDescription(PersistentTasksCustomMetadata.Explanation.NODE_NOT_COMPATIBLE, "not a transform node") + ); } return false; } @@ -243,7 +252,10 @@ public static boolean nodeCanRunThisTransform( if (explain != null) { explain.put( node.getId(), - "transform requires a remote connection but the node does not have the remote_cluster_client role" + new ExplanationAndDescription( + PersistentTasksCustomMetadata.Explanation.REMOTE_NOT_ENABLED, + "transform requires a remote connection but the node does not have the remote_cluster_client role" + ) ); } return false; @@ -252,4 +264,11 @@ public static boolean nodeCanRunThisTransform( // we found no reason that the transform can not run on this node return true; } + + /** + * Container for returning an explanation paired with a detailed description of why an assignment is not possible. + * @param explanation High level enum explanation + * @param description Detailed text description + */ + public record ExplanationAndDescription(PersistentTasksCustomMetadata.Explanation explanation, String description) {} } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java index 279c59b8b712d..79305ebeec6b0 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java @@ -120,7 +120,8 @@ public PersistentTasksCustomMetadata.Assignment getAssignment( if (TransformMetadata.getTransformMetadata(clusterState).isResetMode()) { return new PersistentTasksCustomMetadata.Assignment( null, - "Transform task will not be assigned as a feature reset is in progress." + "Transform task will not be assigned as a feature reset is in progress.", + PersistentTasksCustomMetadata.Explanation.FEATURE_RESET_IN_PROGRESS ); } List unavailableIndices = verifyIndicesPrimaryShardsAreActive(clusterState, resolver); @@ -132,7 +133,11 @@ public PersistentTasksCustomMetadata.Assignment getAssignment( + String.join(",", unavailableIndices) + "]"; logger.debug(reason); - return new PersistentTasksCustomMetadata.Assignment(null, reason); + return new PersistentTasksCustomMetadata.Assignment( + reason, + null, + PersistentTasksCustomMetadata.Explanation.PRIMARY_SHARDS_NOT_ACTIVE + ); } DiscoveryNode discoveryNode = selectLeastLoadedNode( clusterState, @@ -141,21 +146,34 @@ public PersistentTasksCustomMetadata.Assignment getAssignment( ); if (discoveryNode == null) { - Map explainWhyAssignmentFailed = new TreeMap<>(); + Map explainWhyAssignmentFailed = new TreeMap<>(); for (DiscoveryNode node : clusterState.getNodes()) { nodeCanRunThisTransform(node, params.getVersion(), params.requiresRemote(), explainWhyAssignmentFailed); } String reason = "Not starting transform [" + params.getId() + "], reasons [" - + explainWhyAssignmentFailed.entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).collect(Collectors.joining("|")) + + explainWhyAssignmentFailed.entrySet() + .stream() + .map(e -> e.getKey() + ":" + e.getValue().explanation() + " - " + e.getValue().description()) + .collect(Collectors.joining("|")) + "]"; logger.debug(reason); - return new PersistentTasksCustomMetadata.Assignment(null, reason); + return new PersistentTasksCustomMetadata.Assignment( + null, + reason, + explainWhyAssignmentFailed.values() + .stream() + .map(TransformNodes.ExplanationAndDescription::explanation) + .collect(Collectors.toSet()) + ); } - return new PersistentTasksCustomMetadata.Assignment(discoveryNode.getId(), ""); + return new PersistentTasksCustomMetadata.Assignment( + discoveryNode.getId(), + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ); } static List verifyIndicesPrimaryShardsAreActive(ClusterState clusterState, IndexNameExpressionResolver resolver) { diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStopTransformActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStopTransformActionTests.java index 08e0982b2ab84..a1f05bb81a894 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStopTransformActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStopTransformActionTests.java @@ -76,7 +76,10 @@ public void testTaskStateValidationWithTransformTasks() { "non-failed-task", TransformTaskParams.NAME, new TransformTaskParams("transform-task-1", TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("current-data-node-with-1-tasks", "") + new PersistentTasksCustomMetadata.Assignment( + "current-data-node-with-1-tasks", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")).metadata(buildMetadata(pTasksBuilder.build())); @@ -96,7 +99,10 @@ public void testTaskStateValidationWithTransformTasks() { "failed-task", TransformTaskParams.NAME, new TransformTaskParams("transform-task-1", TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("current-data-node-with-1-tasks", "") + new PersistentTasksCustomMetadata.Assignment( + "current-data-node-with-1-tasks", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) .updateTaskState( "failed-task", @@ -128,7 +134,10 @@ public void testTaskStateValidationWithTransformTasks() { "failed-task-2", TransformTaskParams.NAME, new TransformTaskParams("transform-task-2", TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("current-data-node-with-2-tasks", "") + new PersistentTasksCustomMetadata.Assignment( + "current-data-node-with-2-tasks", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) .updateTaskState( "failed-task-2", diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformNodesTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformNodesTests.java index 0f492d4250bc1..f344fde25dd78 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformNodesTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformNodesTests.java @@ -56,13 +56,21 @@ public void testTransformNodes() { transformIdFoo, TransformField.TASK_NAME, new TransformTaskParams(transformIdFoo, TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("node-1", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-1", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( transformIdBar, TransformField.TASK_NAME, new TransformTaskParams(transformIdBar, TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask("test-task1", "testTasks", new PersistentTaskParams() { @Override @@ -84,24 +92,42 @@ public void writeTo(StreamOutput out) { public XContentBuilder toXContent(XContentBuilder builder, Params params) { return null; } - }, new PersistentTasksCustomMetadata.Assignment("node-3", "test assignment")); + }, + new PersistentTasksCustomMetadata.Assignment( + "node-3", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) + ); tasksBuilder.addTask( transformIdFailed, TransformField.TASK_NAME, new TransformTaskParams(transformIdFailed, TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment(null, "awaiting reassignment after node loss") + new PersistentTasksCustomMetadata.Assignment( + null, + "awaiting reassignment after node loss", + PersistentTasksCustomMetadata.Explanation.AWAITING_REASSIGNMENT + ) ); tasksBuilder.addTask( transformIdBaz, TransformField.TASK_NAME, new TransformTaskParams(transformIdBaz, TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("node-2", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-2", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); tasksBuilder.addTask( transformIdOther, TransformField.TASK_NAME, new TransformTaskParams(transformIdOther, TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("node-3", "test assignment") + new PersistentTasksCustomMetadata.Assignment( + "node-3", + "test assignment", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); ClusterState cs = ClusterState.builder(new ClusterName("_name")) @@ -284,7 +310,8 @@ public void testGetAssignment() { ); PersistentTasksCustomMetadata.Assignment assignment2 = new PersistentTasksCustomMetadata.Assignment( randomAlphaOfLengthBetween(1, 10), - randomAlphaOfLengthBetween(1, 10) + randomAlphaOfLengthBetween(1, 10), + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL ); ClusterState clusterState = ClusterState.builder(new ClusterName("some-cluster")) .metadata( diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java index 07801221adc3b..6e3fdd0155cbe 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java @@ -69,6 +69,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -148,8 +150,9 @@ public void testNodeAssignmentProblems() { assertNull(assignment.getExecutorNode()); assertThat( assignment.getExplanation(), - equalTo("Not starting transform [new-task-id], reasons [current-data-node-with-transform-disabled:not a transform node]") + containsString("Not starting transform [new-task-id], reasons [current-data-node-with-transform-disabled:not a transform node]") ); + assertThat(assignment.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.NODE_NOT_COMPATIBLE)); // dedicated transform node nodes = buildNodes(true, false, false, false, true); @@ -185,6 +188,7 @@ public void testNodeAssignmentProblems() { + "]" ) ); + assertThat(assignment.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.CONFIG_VERSION_TOO_LOW)); assignment = executor.getAssignment( new TransformTaskParams("new-task-id", TransformConfigVersion.V_7_5_0, null, false), @@ -214,6 +218,7 @@ public void testNodeAssignmentProblems() { + "]" ) ); + assertThat(assignment.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.REMOTE_NOT_ENABLED)); assignment = executor.getAssignment( new TransformTaskParams("new-task-id", TransformConfigVersion.CURRENT, null, false), @@ -245,6 +250,8 @@ public void testNodeAssignmentProblems() { + "]" ) ); + assertThat(assignment.getExplanationCodes(), contains(PersistentTasksCustomMetadata.Explanation.NODE_NOT_COMPATIBLE)); + // old node, we do not know if remote is enabled nodes = buildNodes(false, true, false, true, false); cs = buildClusterState(nodes); @@ -518,19 +525,28 @@ private ClusterState buildClusterState(DiscoveryNodes.Builder nodes) { "transform-task-1", TransformTaskParams.NAME, new TransformTaskParams("transform-task-1", TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("current-data-node-with-1-tasks", "") + new PersistentTasksCustomMetadata.Assignment( + "current-data-node-with-1-tasks", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) .addTask( "transform-task-2", TransformTaskParams.NAME, new TransformTaskParams("transform-task-2", TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("current-data-node-with-2-tasks", "") + new PersistentTasksCustomMetadata.Assignment( + "current-data-node-with-2-tasks", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ) .addTask( "transform-task-3", TransformTaskParams.NAME, new TransformTaskParams("transform-task-3", TransformConfigVersion.CURRENT, null, false), - new PersistentTasksCustomMetadata.Assignment("current-data-node-with-2-tasks", "") + new PersistentTasksCustomMetadata.Assignment( + "current-data-node-with-2-tasks", + PersistentTasksCustomMetadata.Explanation.ASSIGNMENT_SUCCESSFUL + ) ); PersistentTasksCustomMetadata pTasks = pTasksBuilder.build();