Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/133916.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 133916
summary: Fix model assignment error handling and assignment explanation generation
area: Machine Learning
type: bug
issues: []
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,9 @@ static void copyAssignments(AssignmentPlan source, AssignmentPlan.Builder dest,
Map<AssignmentPlan.Node, Integer> sourceNodeAssignments = source.assignments(deployment).orElse(Map.of());
for (Map.Entry<AssignmentPlan.Node, Integer> sourceAssignment : sourceNodeAssignments.entrySet()) {
AssignmentPlan.Node node = originalNodeById.get(sourceAssignment.getKey().id());
dest.assignModelToNode(deployment, node, sourceAssignment.getValue());
if (dest.canAssign(deployment, node, sourceAssignment.getValue())) {
dest.assignModelToNode(deployment, node, sourceAssignment.getValue());
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,7 @@ int getRemainingAllocations(Deployment m) {
return remainingModelAllocations.get(m);
}

boolean canAssign(Deployment deployment, Node node, int allocations) {
public boolean canAssign(Deployment deployment, Node node, int allocations) {
long requiredMemory = getDeploymentMemoryRequirement(deployment, node, allocations);
return canAssign(deployment, node, allocations, requiredMemory);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,9 @@ private AssignmentPlan swapOriginalDeploymentsInPlan(
Map<Node, Integer> nodeAssignments = plan.assignments(planDeployment).orElse(Map.of());
for (Map.Entry<Node, Integer> assignment : nodeAssignments.entrySet()) {
Node originalNode = originalNodeById.get(assignment.getKey().id());
finalPlanBuilder.assignModelToNode(originalDeployment, originalNode, assignment.getValue());
if (finalPlanBuilder.canAssign(originalDeployment, originalNode, assignment.getValue())) {
finalPlanBuilder.assignModelToNode(originalDeployment, originalNode, assignment.getValue());
}
}
}
return finalPlanBuilder.build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1198,7 +1198,7 @@ public void testCopyAssignments() {
assertThat(deployment2Assignments.get().get(node2), equalTo(1));
}

public void testRebalance_GivenDeploymentWithMemoryRequirements_ConsidersNativeExecutableOverhead() {
public void testRebalance_GivenDeploymentWithMemoryRequirements_ExplainMissingAllocations() {
// Create a node with just enough memory to fit the model plus native executable overhead
long modelMemory = ByteSizeValue.ofMb(200).getBytes();
long memoryOverhead = ByteSizeValue.ofMb(240).getBytes();
Expand Down