Stop and resume an explainer (#706)

* Stop and resume an explainer (kserve#4546)

Signed-off-by: Hannah DeFazio <[email protected]>

* Whitespace

Signed-off-by: Hannah DeFazio <[email protected]>

---------

Signed-off-by: Hannah DeFazio <[email protected]>

Author: hdefazio

pkg/controller/v1beta1/inferenceservice/components/explainer.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
+	"knative.dev/pkg/apis"
 	knservingv1 "knative.dev/serving/pkg/apis/serving/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -149,6 +150,7 @@ func (e *Explainer) Reconcile(ctx context.Context, isvc *v1beta1.InferenceServic
 			explainerAnnotations,
 		),
 	}
+
 	container := explainer.GetContainer(isvc.ObjectMeta, isvc.Spec.Explainer.GetExtensions(), e.inferenceServiceConfig, predictorName)
 	if len(isvc.Spec.Explainer.PodSpec.Containers) == 0 {
 		isvc.Spec.Explainer.PodSpec.Containers = []corev1.Container{
@@ -162,46 +164,79 @@ func (e *Explainer) Reconcile(ctx context.Context, isvc *v1beta1.InferenceServic
 
 	// Here we allow switch between knative and vanilla deployment
 	if e.deploymentMode == constants.RawDeployment {
-		r, err := raw.NewRawKubeReconciler(ctx, e.client, e.clientset, e.scheme, constants.InferenceServiceResource, objectMeta, metav1.ObjectMeta{},
-			&isvc.Spec.Explainer.ComponentExtensionSpec, &podSpec, nil)
-		if err != nil {
-			return ctrl.Result{}, errors.Wrapf(err, "fails to create NewRawKubeReconciler for explainer")
-		}
-		// set Deployment Controller
-		for _, deployment := range r.Deployment.DeploymentList {
-			if err := controllerutil.SetControllerReference(isvc, deployment, e.scheme); err != nil {
-				return ctrl.Result{}, errors.Wrapf(err, "fails to set deployment owner reference for explainer")
-			}
-		}
-		// set Service Controller
-		for _, svc := range r.Service.ServiceList {
-			if err := controllerutil.SetControllerReference(isvc, svc, e.scheme); err != nil {
-				return ctrl.Result{}, errors.Wrapf(err, "fails to set service owner reference for explainer")
-			}
+		if err := e.reconcileExplainerRawDeployment(ctx, isvc, &objectMeta, &podSpec); err != nil {
+			return ctrl.Result{}, err
 		}
-		// set autoscaler Controller
-		if err := r.Scaler.Autoscaler.SetControllerReferences(isvc, e.scheme); err != nil {
-			return ctrl.Result{}, errors.Wrapf(err, "fails to set autoscaler owner references for explainer")
+	} else {
+		if err := e.reconcileExplainerKnativeDeployment(ctx, isvc, &objectMeta, &podSpec); err != nil {
+			return ctrl.Result{}, err
 		}
+	}
 
-		deployment, err := r.Reconcile(ctx)
-		if err != nil {
-			return ctrl.Result{}, errors.Wrapf(err, "fails to reconcile explainer")
+	if utils.GetForceStopRuntime(isvc) {
+		// Exit early if we have already set the explainer's status to stopped
+		existingExplainerCondition := isvc.Status.GetCondition(v1beta1.ExplainerReady)
+		if existingExplainerCondition != nil && existingExplainerCondition.Status == corev1.ConditionFalse && existingExplainerCondition.Reason == v1beta1.StoppedISVCReason {
+			return ctrl.Result{}, nil
 		}
-		isvc.Status.PropagateRawStatus(v1beta1.ExplainerComponent, deployment, r.URL)
-	} else {
-		knutils.ValidateInitialScaleAnnotation(objectMeta.Annotations, e.allowZeroInitialScale, isvc.Spec.Explainer.MinReplicas, e.Log)
-		r := knative.NewKsvcReconciler(e.client, e.scheme, objectMeta, &isvc.Spec.Explainer.ComponentExtensionSpec,
-			&podSpec, isvc.Status.Components[v1beta1.ExplainerComponent], e.inferenceServiceConfig.ServiceLabelDisallowedList)
 
-		if err := controllerutil.SetControllerReference(isvc, r.Service, e.scheme); err != nil {
-			return ctrl.Result{}, errors.Wrapf(err, "fails to set owner reference for explainer")
+		// Set the ready condition to false
+		isvc.Status.SetCondition(v1beta1.ExplainerReady, &apis.Condition{
+			Type:   v1beta1.ExplainerReady,
+			Status: corev1.ConditionFalse,
+			Reason: v1beta1.StoppedISVCReason,
+		})
+	}
+	return ctrl.Result{}, nil
+}
+
+func (e *Explainer) reconcileExplainerRawDeployment(ctx context.Context, isvc *v1beta1.InferenceService, objectMeta *metav1.ObjectMeta, podSpec *corev1.PodSpec) error {
+	r, err := raw.NewRawKubeReconciler(ctx, e.client, e.clientset, e.scheme, constants.InferenceServiceResource, *objectMeta, metav1.ObjectMeta{},
+		&isvc.Spec.Explainer.ComponentExtensionSpec, podSpec, nil)
+	if err != nil {
+		return errors.Wrapf(err, "fails to create NewRawKubeReconciler for explainer")
+	}
+	// set Deployment Controller
+	for _, deployment := range r.Deployment.DeploymentList {
+		if err := controllerutil.SetControllerReference(isvc, deployment, e.scheme); err != nil {
+			return errors.Wrapf(err, "fails to set deployment owner reference for explainer")
 		}
-		status, err := r.Reconcile(ctx)
-		if err != nil {
-			return ctrl.Result{}, errors.Wrapf(err, "fails to reconcile explainer")
+	}
+	// set Service Controller
+	for _, svc := range r.Service.ServiceList {
+		if err := controllerutil.SetControllerReference(isvc, svc, e.scheme); err != nil {
+			return errors.Wrapf(err, "fails to set service owner reference for explainer")
 		}
+	}
+	// set autoscaler Controller
+	if err := r.Scaler.Autoscaler.SetControllerReferences(isvc, e.scheme); err != nil {
+		return errors.Wrapf(err, "fails to set autoscaler owner references for explainer")
+	}
+
+	deployment, err := r.Reconcile(ctx)
+	if err != nil {
+		return errors.Wrapf(err, "fails to reconcile explainer")
+	}
+	if !utils.GetForceStopRuntime(isvc) {
+		isvc.Status.PropagateRawStatus(v1beta1.ExplainerComponent, deployment, r.URL)
+	}
+	return nil
+}
+
+func (e *Explainer) reconcileExplainerKnativeDeployment(ctx context.Context, isvc *v1beta1.InferenceService, objectMeta *metav1.ObjectMeta, podSpec *corev1.PodSpec) error {
+	knutils.ValidateInitialScaleAnnotation(objectMeta.Annotations, e.allowZeroInitialScale, isvc.Spec.Explainer.MinReplicas, e.Log)
+	r := knative.NewKsvcReconciler(e.client, e.scheme, *objectMeta, &isvc.Spec.Explainer.ComponentExtensionSpec,
+		podSpec, isvc.Status.Components[v1beta1.ExplainerComponent], e.inferenceServiceConfig.ServiceLabelDisallowedList)
+
+	if err := controllerutil.SetControllerReference(isvc, r.Service, e.scheme); err != nil {
+		return errors.Wrapf(err, "fails to set owner reference for explainer")
+	}
+	status, err := r.Reconcile(ctx)
+	if err != nil {
+		return errors.Wrapf(err, "fails to reconcile explainer")
+	}
+	if !utils.GetForceStopRuntime(isvc) {
 		isvc.Status.PropagateStatus(v1beta1.ExplainerComponent, status)
 	}
-	return ctrl.Result{}, nil
+	return nil
 }