Subnet claim and subnet observation for removal of scheduling gates from instances, instance status updates when programmed.

Author: joshlreese

api/v1alpha/instance_types.go

@@ -383,11 +383,23 @@ type InstanceControllerStatus struct {
 const (
 	// InstanceReady indicates that the instance is ready
 	InstanceReady = "Ready"
+
+	// InstanceProgrammed indicates that the instance has been programmed
+	InstanceProgrammed = "Programmed"
 )
 
 const (
 	// InstanceReadyReasonSchedulingGatesPresent indicates that the instance is not ready because scheduling gates are present.
 	InstanceReadyReasonSchedulingGatesPresent = "SchedulingGatesPresent"
+
+	// InstanceProgrammedReasonNotProgrammed indicates that the instance has not been programmed
+	InstanceProgrammedReasonNotProgrammed = "NotProgrammed"
+
+	// InstanceProgrammedReasonProgrammingInProgress indicates that the instance is being programmed.
+	InstanceProgrammedReasonProgrammingInProgress = "ProgrammingInProgress"
+
+	// InstanceProgrammedReasonProgrammed indicates that the instance has been programmed
+	InstanceProgrammedReasonProgrammed = "Programmed"
 )
 
 type InstanceTemplateSpec struct {
@@ -419,7 +431,7 @@ type Instance struct {
 
 	// Status defines the current state of an Instance.
 	//
-	// +kubebuilder:default={conditions:{{type:"Running",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}}
+	// +kubebuilder:default={conditions:{{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Running",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}}
 	Status InstanceStatus `json:"status,omitempty"`
 }
 

cmd/main.go

@@ -178,13 +178,13 @@ func main() {
 		os.Exit(1)
 	}
 
+	// +kubebuilder:scaffold:builder
+
 	if err = controller.AddIndexers(ctx, mgr); err != nil {
 		setupLog.Error(err, "unable to add indexers")
 		os.Exit(1)
 	}
 
-	// +kubebuilder:scaffold:builder
-
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)

config/crd/bases/compute.datumapis.com_instances.yaml

@@ -811,6 +811,11 @@ spec:
           status:
             default:
               conditions:
+              - lastTransitionTime: "1970-01-01T00:00:00Z"
+                message: Waiting for controller
+                reason: Pending
+                status: Unknown
+                type: Programmed
               - lastTransitionTime: "1970-01-01T00:00:00Z"
                 message: Waiting for controller
                 reason: Pending

internal/controller/indexers.go

@@ -15,6 +15,7 @@ import (
 const (
 	deploymentWorkloadUIDIndex = "deploymentWorkloadUIDIndex"
 	workloadNetworksIndex      = "workloadNetworksIndex"
+	deploymentLocationIndex    = "deploymentLocationIndex"
 )
 
 func AddIndexers(ctx context.Context, mgr mcmanager.Manager) error {
@@ -29,6 +30,11 @@ func addWorkloadDeploymentIndexers(ctx context.Context, mgr mcmanager.Manager) e
 		return fmt.Errorf("failed to add workload deployment indexer %q: %w", deploymentWorkloadUIDIndex, err)
 	}
 
+	// Index workload deployments by location
+	if err := mgr.GetFieldIndexer().IndexField(ctx, &computev1alpha.WorkloadDeployment{}, deploymentLocationIndex, deploymentLocationIndexFunc); err != nil {
+		return fmt.Errorf("failed to add workload deployment indexer %q: %w", deploymentLocationIndex, err)
+	}
+
 	return nil
 }
 
@@ -38,6 +44,20 @@ func deploymentWorkloadUIDIndexFunc(o client.Object) []string {
 	}
 }
 
+func deploymentLocationIndexFunc(o client.Object) []string {
+	deployment := o.(*computev1alpha.WorkloadDeployment)
+	if deployment.Status.Location == nil {
+		return nil
+	}
+
+	return []string{
+		types.NamespacedName{
+			Namespace: deployment.Status.Location.Namespace,
+			Name:      deployment.Status.Location.Name,
+		}.String(),
+	}
+}
+
 func addWorkloadIndexers(ctx context.Context, mgr mcmanager.Manager) error {
 	if err := mgr.GetFieldIndexer().IndexField(ctx, &computev1alpha.Workload{}, workloadNetworksIndex, workloadNetworksIndexFunc); err != nil {
 		return fmt.Errorf("failed to add workload indexer %q: %w", workloadNetworksIndex, err)

internal/controller/instance_controller.go

@@ -11,6 +11,7 @@ import (
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder"
 	mccontext "sigs.k8s.io/multicluster-runtime/pkg/context"
@@ -53,6 +54,8 @@ func (r *InstanceReconciler) Reconcile(ctx context.Context, req mcreconcile.Requ
 	logger.Info("reconciling instance")
 	defer logger.Info("reconcile complete")
 
+	// TODO(jreese) better condition handling
+
 	if len(instance.Spec.Controller.SchedulingGates) > 0 {
 		// Update Ready condition to False, Reason to "SchedulingGatesPresent"
 		// and Message to "Scheduling gates present"
@@ -63,19 +66,58 @@ func (r *InstanceReconciler) Reconcile(ctx context.Context, req mcreconcile.Requ
 			schedulingGateNames = append(schedulingGateNames, gate.Name)
 		}
 
-		changed := apimeta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
+		if _, err := controllerutil.CreateOrPatch(ctx, cl.GetClient(), &instance, func() error {
+			apimeta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
+				Type:               computev1alpha.InstanceReady,
+				Status:             metav1.ConditionFalse,
+				Reason:             computev1alpha.InstanceReadyReasonSchedulingGatesPresent,
+				Message:            fmt.Sprintf("Scheduling gates present: %s", strings.Join(schedulingGateNames, ", ")),
+				ObservedGeneration: instance.Generation,
+			})
+			return nil
+		}); err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed updating instance status: %w", err)
+		}
+
+		return ctrl.Result{}, nil
+	}
+
+	if condition := apimeta.FindStatusCondition(instance.Status.Conditions, computev1alpha.InstanceProgrammed); condition.Status != metav1.ConditionTrue {
+		logger.Info("instance is not programmed", "instance", instance.Name)
+		message := "Instance has not been programmed"
+		if condition.Status != metav1.ConditionUnknown {
+			message = condition.Message
+		}
+
+		logger.Info("updating instance status", "instance", instance.Name)
+		if _, err := controllerutil.CreateOrPatch(ctx, cl.GetClient(), &instance, func() error {
+			apimeta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
+				Type:               computev1alpha.InstanceReady,
+				Status:             metav1.ConditionFalse,
+				Reason:             computev1alpha.InstanceProgrammedReasonNotProgrammed,
+				Message:            message,
+				ObservedGeneration: instance.Generation,
+			})
+			return nil
+		}); err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed updating instance status: %w", err)
+		}
+		return ctrl.Result{}, nil
+	}
+
+	logger.Info("instance is programmed", "instance", instance.Name)
+
+	if _, err := controllerutil.CreateOrPatch(ctx, cl.GetClient(), &instance, func() error {
+		apimeta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
 			Type:               computev1alpha.InstanceReady,
-			Status:             metav1.ConditionFalse,
-			Reason:             computev1alpha.InstanceReadyReasonSchedulingGatesPresent,
-			Message:            fmt.Sprintf("Scheduling gates present: %s", strings.Join(schedulingGateNames, ", ")),
+			Status:             metav1.ConditionTrue,
+			Reason:             computev1alpha.InstanceProgrammedReasonProgrammed,
+			Message:            "Instance is ready",
 			ObservedGeneration: instance.Generation,
 		})
-
-		if changed {
-			if err := cl.GetClient().Status().Update(ctx, &instance); err != nil {
-				return ctrl.Result{}, fmt.Errorf("failed updating instance status: %w", err)
-			}
-		}
+		return nil
+	}); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed updating instance status: %w", err)
 	}
 
 	return ctrl.Result{}, nil

internal/controller/workloaddeployment_controller.go

@@ -10,10 +10,14 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder"
 	mccontext "sigs.k8s.io/multicluster-runtime/pkg/context"
 	mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager"
@@ -110,6 +114,7 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco
 	// gates on instances. If any instances were created by actions above, those
 	// will result in this reconciler being processed again, so will properly have
 	// their gates removed.
+	logger.Info("removing scheduling gates from instances")
 
 	for _, instance := range instances.Items {
 		if len(instance.Spec.Controller.SchedulingGates) == 0 {
@@ -125,10 +130,10 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco
 			continue
 		}
 
-		instance.Spec.Controller.SchedulingGates = newGates
-
-		// TODO(jreese) consider using patches
-		if err := cl.GetClient().Update(ctx, &instance); err != nil {
+		if _, err := controllerutil.CreateOrPatch(ctx, cl.GetClient(), &instance, func() error {
+			instance.Spec.Controller.SchedulingGates = newGates
+			return nil
+		}); err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed updating instance: %w", err)
 		}
 	}
@@ -318,7 +323,7 @@ func (r *WorkloadDeploymentReconciler) reconcileNetworks(
 			return result
 		}
 
-		logger.Info("found subnet", "subnet", subnet.Name)
+		logger.Info("subnet is ready", "subnet", subnet.Name)
 
 	}
 
@@ -329,11 +334,62 @@ func (r *WorkloadDeploymentReconciler) reconcileNetworks(
 func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager) error {
 	r.mgr = mgr
 	// TODO(jreese) finalizers
-	// TODO(jreese) watch subnet claims and subnets, enqueue based on location
-	// match on workload deployment.
 	return mcbuilder.ControllerManagedBy(mgr).
 		For(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)).
 		Owns(&computev1alpha.Instance{}).
 		Owns(&networkingv1alpha.NetworkBinding{}).
+		Watches(&networkingv1alpha.SubnetClaim{}, func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] {
+			return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request {
+				subnetClaim := o.(*networkingv1alpha.SubnetClaim)
+				return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnetClaim.Spec.Location)
+			})
+		}).
+		Watches(&networkingv1alpha.Subnet{}, func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] {
+			return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request {
+				subnet := o.(*networkingv1alpha.Subnet)
+				return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnet.Spec.Location)
+			})
+		}).
 		Complete(r)
 }
+
+func enqueueWorkloadDeploymentByLocation(ctx context.Context, mgr mcmanager.Manager, clusterName string, locationRef networkingv1alpha.LocationReference) []mcreconcile.Request {
+	logger := log.FromContext(ctx)
+
+	cluster, err := mgr.GetCluster(ctx, clusterName)
+	if err != nil {
+		logger.Error(err, "failed to get cluster")
+		return nil
+	}
+	clusterClient := cluster.GetClient()
+
+	locationName := (types.NamespacedName{
+		Namespace: locationRef.Namespace,
+		Name:      locationRef.Name,
+	}).String()
+	listOpts := client.MatchingFields{
+		deploymentLocationIndex: locationName,
+	}
+
+	var workloadDeployments computev1alpha.WorkloadDeploymentList
+
+	if err := clusterClient.List(ctx, &workloadDeployments, listOpts); err != nil {
+		logger.Error(err, "failed to list workloads")
+		return nil
+	}
+
+	var requests []mcreconcile.Request
+	for _, workload := range workloadDeployments.Items {
+		requests = append(requests, mcreconcile.Request{
+			Request: reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: workload.Namespace,
+					Name:      workload.Name,
+				},
+			},
+			ClusterName: clusterName,
+		})
+	}
+
+	return requests
+}