add autopilotExempt annotation to allow per-AppWrapper opt-out (#203)

Author: dgrove-oss

api/v1beta2/appwrapper_types.go

@@ -174,6 +174,7 @@ const (
 	SuccessTTLAnnotation                   = "workload.codeflare.dev.appwrapper/successTTLDuration"
 	TerminalExitCodesAnnotation            = "workload.codeflare.dev.appwrapper/terminalExitCodes"
 	RetryableExitCodesAnnotation           = "workload.codeflare.dev.appwrapper/retryableExitCodes"
+	AutopilotExemptAnnotation              = "workload.codeflare.dev.appwrapper/autopilotExempt"
 )
 
 //+kubebuilder:object:root=true

internal/controller/appwrapper/appwrapper_controller.go

@@ -510,7 +510,7 @@ func (r *AppWrapperReconciler) getPodStatus(ctx context.Context, aw *workloadv1b
 		return nil, err
 	}
 	summary := &podStatusSummary{expected: pc}
-	checkUnhealthyNodes := r.Config.Autopilot != nil && r.Config.Autopilot.EvacuateWorkloads
+	checkUnhealthyNodes := r.Config.Autopilot != nil && r.Config.Autopilot.MigrateImpactedWorkloads && !r.isAutopilotExempt(ctx, aw)
 
 	for _, pod := range pods.Items {
 		switch pod.Status.Phase {
@@ -855,6 +855,17 @@ func (r *AppWrapperReconciler) retryableExitCodes(_ context.Context, aw *workloa
 	return ans
 }
 
+func (r *AppWrapperReconciler) isAutopilotExempt(ctx context.Context, aw *workloadv1beta2.AppWrapper) bool {
+	if v, ok := aw.Annotations[workloadv1beta2.AutopilotExemptAnnotation]; ok {
+		if isExempt, err := strconv.ParseBool(v); err == nil {
+			return isExempt
+		} else {
+			log.FromContext(ctx).Error(err, "Malformed autopilotExempt annotation; treating as false", "annotation", v)
+		}
+	}
+	return false
+}
+
 func clearCondition(aw *workloadv1beta2.AppWrapper, condition workloadv1beta2.AppWrapperCondition, reason string, message string) {
 	if meta.IsStatusConditionTrue(aw.Status.Conditions, string(condition)) {
 		meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{

internal/controller/appwrapper/resource_management.go

@@ -252,7 +252,7 @@ func (r *AppWrapperReconciler) createComponent(ctx context.Context, aw *workload
 			}
 		}
 
-		if r.Config.Autopilot != nil && r.Config.Autopilot.InjectAffinity {
+		if r.Config.Autopilot != nil && r.Config.Autopilot.InjectAntiAffinities && !r.isAutopilotExempt(ctx, aw) {
 			toAdd := map[string]string{}
 			for resource, labels := range r.Config.Autopilot.ResourceUnhealthyConfig {
 				if hasResourceRequest(spec, resource) {

pkg/config/config.go

@@ -47,9 +47,9 @@ type KueueJobReconcillerConfig struct {
 }
 
 type AutopilotConfig struct {
-	InjectAffinity          bool                         `json:"injectAffinity,omitempty"`
-	EvacuateWorkloads       bool                         `json:"evacuateWorkloads,omitempty"`
-	ResourceUnhealthyConfig map[string]map[string]string `json:"resourceUnhealthyConfig,omitempty"`
+	InjectAntiAffinities     bool                         `json:"injectAntiAffinities,omitempty"`
+	MigrateImpactedWorkloads bool                         `json:"migrateImpactedWorkloads,omitempty"`
+	ResourceUnhealthyConfig  map[string]map[string]string `json:"resourceUnhealthyConfig,omitempty"`
 }
 
 type FaultToleranceConfig struct {
@@ -100,8 +100,8 @@ func NewAppWrapperConfig() *AppWrapperConfig {
 			LabelKeysToCopy:            []string{},
 		},
 		Autopilot: &AutopilotConfig{
-			InjectAffinity:    true,
-			EvacuateWorkloads: true,
+			InjectAntiAffinities:     true,
+			MigrateImpactedWorkloads: true,
 			ResourceUnhealthyConfig: map[string]map[string]string{
 				"nvidia.com/gpu": {"autopilot.ibm.com/gpuhealth": "ERR"},
 			},

pkg/controller/setup.go

@@ -50,7 +50,7 @@ func SetupControllers(mgr ctrl.Manager, awConfig *config.AppWrapperConfig) error
 		}
 	}
 
-	if awConfig.Autopilot != nil && awConfig.Autopilot.EvacuateWorkloads {
+	if awConfig.Autopilot != nil && awConfig.Autopilot.MigrateImpactedWorkloads {
 		if err := (&appwrapper.NodeHealthMonitor{
 			Client: mgr.GetClient(),
 			Config: awConfig,