Updating Knative Configuration and Annotations to Support 0 Initial Scale (#537) (#579)

* adding initial-scale and max-scale annotations to knative service resources created for inference graphs



* adding initial-scale annotation for knative service resources created for inference services



* now checking knative autoscaler configuration prior to knative service creation



* fixing golang linter errors



* now relying only on the knativeserving cr to read the autoscaler configuration



* addressing comments



* updating comments



* adding knative operator APIs to scheme and using kubebuilder to set kserve-manager-role rules



* reformatting



* updating GetAutoscalerConfiguration method to look for both the autoscale and config-autoscaler keys in the KnativeServing CR



* refactoring GetAutoscalerConfiguration method



---------

Signed-off-by: Brett Thompson <[email protected]>

Author: brettmthompson

charts/kserve-resources/templates/clusterrole.yaml

@@ -132,6 +132,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - operator.knative.dev
+  resources:
+  - knativeservings
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - rbac.authorization.k8s.io
   resourceNames:
@@ -149,6 +157,7 @@ rules:
   - routes
   verbs:
   - create
+  - delete
   - get
   - list
   - patch

cmd/manager/main.go

@@ -31,6 +31,7 @@ import (
 	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/client-go/tools/record"
+	operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1"
 	knservingv1 "knative.dev/serving/pkg/apis/serving/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -176,6 +177,20 @@ func main() {
 			os.Exit(1)
 		}
 	}
+
+	knServingFound, knServingCheckErr := utils.IsCrdAvailable(cfg, operatorv1beta1.SchemeGroupVersion.String(), constants.KnativeServingKind)
+	if knServingCheckErr != nil {
+		setupLog.Error(knServingCheckErr, "error when checking if Knative KnativeServing kind is available")
+		os.Exit(1)
+	}
+	if knServingFound {
+		setupLog.Info("Setting up Knative Operator scheme")
+		if err := operatorv1beta1.AddToScheme(mgr.GetScheme()); err != nil {
+			setupLog.Error(err, "unable to add Knative Operator APIs to scheme")
+			os.Exit(1)
+		}
+	}
+
 	if !ingressConfig.DisableIstioVirtualHost {
 		vsFound, vsCheckErr := utils.IsCrdAvailable(cfg, istioclientv1beta1.SchemeGroupVersion.String(), constants.IstioVirtualServiceKind)
 		if vsCheckErr != nil {

config/configmap/inferenceservice.yaml

@@ -59,6 +59,7 @@ data:
         # revisions, which prevents the reconciliation loop to be triggered if the annotations is 
         # configured here are used.
         # Default values are:
+        #  "autoscaling.knative.dev/initial-scale",
         #  "autoscaling.knative.dev/min-scale",
         #  "autoscaling.knative.dev/max-scale",
         #  "internal.serving.kserve.io/storage-initializer-sourceuri",

config/overlays/odh/inferenceservice-config-patch.yaml

@@ -88,6 +88,7 @@ data:
   inferenceService: |-
     {
       "serviceAnnotationDisallowedList": [
+        "autoscaling.knative.dev/initial-scale",
         "autoscaling.knative.dev/min-scale",
         "autoscaling.knative.dev/max-scale",
         "internal.serving.kserve.io/storage-initializer-sourceuri",

config/rbac/role.yaml

@@ -119,6 +119,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - operator.knative.dev
+  resources:
+  - knativeservings
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - rbac.authorization.k8s.io
   resourceNames:

go.mod

@@ -40,6 +40,7 @@ require (
 	k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2
 	k8s.io/utils v0.0.0-20240821151609-f90d01438635
 	knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b
+	knative.dev/operator v0.42.2
 	knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c
 	knative.dev/serving v0.42.2
 	sigs.k8s.io/controller-runtime v0.19.1

go.sum

@@ -286,6 +286,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/manifestival/manifestival v0.7.2 h1:l4uFdWX/xQK4QcRfqGoMtBvaZeWPEuwD6hVsCwUqZY4=
+github.com/manifestival/manifestival v0.7.2/go.mod h1:nl3T6HlfHCeidooWVTMI9vYNTBkQ1GdhLNb+smozbdk=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -765,8 +767,12 @@ k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 h1:GKE9U8BH16uynoxQii0auT
 k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA=
 k8s.io/utils v0.0.0-20240821151609-f90d01438635 h1:2wThSvJoW/Ncn9TmQEYXRnevZXi2duqHWf5OX9S3zjI=
 k8s.io/utils v0.0.0-20240821151609-f90d01438635/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+knative.dev/caching v0.0.0-20240716132144-989f54c83776 h1:2nINnWuXtb9e2nG/EJxSCeghcmu6qmvmomJ7woiP5Is=
+knative.dev/caching v0.0.0-20240716132144-989f54c83776/go.mod h1:Uj74eO9rLiK1eb8wmDBED1hJBZQ7MJ9cvq/d8Ktsm3c=
 knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b h1:ws/Jeho6on84+5tfNKLAKriVVGIwivHbgPEtZjBfcs0=
 knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b/go.mod h1:2eMQVGLBZ5Kj1C4kKPuPhO7BsUeF6fkmhZFDQPIP+88=
+knative.dev/operator v0.42.2 h1:wgAWYHwoSFmV+wPHCt5dZahHTHLy2VCM4G82PEo9iSc=
+knative.dev/operator v0.42.2/go.mod h1:cfSpJMgvwmuZ7USaxC+zgEuizMFc/xweREW5DG6J1DA=
 knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c h1:2crXVk4FG0dSG6WHaIT+WKbUzn7qG2wn0AfYmvA22zs=
 knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c/go.mod h1:cI2RPEEHZk+/dBpfHobs0aBdPA1mMZVUVWnGAc8NSzM=
 knative.dev/serving v0.42.2 h1:yKieg3MeNvpVz+4JJPbvmpee3v3LK3zO5h5HJBtzaNk=

pkg/constants/constants.go

@@ -90,6 +90,7 @@ var (
 	AutoscalerClass                             = KServeAPIGroupName + "/autoscalerClass"
 	AutoscalerMetrics                           = KServeAPIGroupName + "/metrics"
 	TargetUtilizationPercentage                 = KServeAPIGroupName + "/targetUtilizationPercentage"
+	InitialScaleAnnotationKey                   = KnativeAutoscalingAPIGroupName + "/initial-scale"
 	MinScaleAnnotationKey                       = KnativeAutoscalingAPIGroupName + "/min-scale"
 	MaxScaleAnnotationKey                       = KnativeAutoscalingAPIGroupName + "/max-scale"
 	RollOutDurationAnnotationKey                = KnativeServingAPIGroupName + "/rollout-duration"
@@ -257,9 +258,14 @@ type InferenceServiceProtocol string
 
 // Knative constants
 const (
-	KnativeLocalGateway   = "knative-serving/knative-local-gateway"
-	KnativeIngressGateway = "knative-serving/knative-ingress-gateway"
-	VisibilityLabel       = "networking.knative.dev/visibility"
+	AutoscalerKey               = "autoscaler"
+	AutoscalerInitialScaleKey   = "initial-scale"
+	AutoscalerAllowZeroScaleKey = "allow-zero-initial-scale"
+	DefaultKnServingName        = "knative-serving"
+	DefaultKnServingNamespace   = "knative-serving"
+	KnativeLocalGateway         = "knative-serving/knative-local-gateway"
+	KnativeIngressGateway       = "knative-serving/knative-ingress-gateway"
+	VisibilityLabel             = "networking.knative.dev/visibility"
 )
 
 var (
@@ -374,6 +380,7 @@ var (
 	// revisions, which prevents the reconciliation loop to be triggered if the annotations is
 	// configured here are used.
 	ServiceAnnotationDisallowedList = []string{
+		autoscaling.InitialScaleAnnotationKey,
 		autoscaling.MinScaleAnnotationKey,
 		autoscaling.MaxScaleAnnotationKey,
 		StorageInitializerSourceUriInternalAnnotationKey,
@@ -499,6 +506,7 @@ const (
 	IstioVirtualServiceKind = "VirtualService"
 	KnativeServiceKind      = "Service"
 	ClusterLocalModelKind   = "ClusterLocalModel"
+	KnativeServingKind      = "KnativeServing"
 )
 
 // Model Parallel Options

pkg/controller/v1alpha1/inferencegraph/controller.go

@@ -23,6 +23,7 @@ limitations under the License.
 // +kubebuilder:rbac:groups=serving.knative.dev,resources=services/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=create;get;update;patch;watch;delete
 // +kubebuilder:rbac:groups=route.openshift.io,resources=routes/status,verbs=get
+// +kubebuilder:rbac:groups=operator.knative.dev,resources=knativeservings,verbs=get;list;watch
 package inferencegraph
 
 import (
@@ -45,6 +46,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
+	operatorv1beta1 "knative.dev/operator/pkg/apis/operator/v1beta1"
 	"knative.dev/pkg/apis"
 	knservingv1 "knative.dev/serving/pkg/apis/serving/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -267,10 +269,26 @@ func (r *InferenceGraphReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		if !ksvcAvailable {
 			r.Recorder.Event(graph, v1.EventTypeWarning, "ServerlessModeRejected",
 				"It is not possible to use Serverless deployment mode when Knative Services are not available")
-			return reconcile.Result{Requeue: false}, reconcile.TerminalError(fmt.Errorf("the resolved deployment mode of InferenceGraph '%s' is Serverless, but Knative Serving is not available", graph.Name))
+			return reconcile.Result{Requeue: false}, reconcile.TerminalError(fmt.Errorf("the resolved deployment mode of InferenceGraph '%s' is Serverless, but Knative Services are not available", graph.Name))
+		}
+
+		// Abort if Knative KnativeServings are not available
+		knServingFound, knServingCheckErr := utils.IsCrdAvailable(r.ClientConfig, operatorv1beta1.SchemeGroupVersion.String(), constants.KnativeServingKind)
+		if knServingCheckErr != nil {
+			return reconcile.Result{}, knServingCheckErr
+		}
+
+		if !knServingFound {
+			r.Recorder.Event(graph, v1.EventTypeWarning, "ServerlessModeRejected",
+				"It is not possible to use Serverless deployment mode when Knative KnativeServings are not available")
+			return reconcile.Result{Requeue: false}, reconcile.TerminalError(fmt.Errorf("the resolved deployment mode of InferenceGraph '%s' is Serverless, but Knative KnativeServings are not available", graph.Name))
+		}
+
+		desired, err := createKnativeService(r.Client, graph.ObjectMeta, graph, routerConfig)
+		if err != nil {
+			return ctrl.Result{}, errors.Wrapf(err, "fails to create new knative service")
 		}
 
-		desired := createKnativeService(graph.ObjectMeta, graph, routerConfig)
 		err = controllerutil.SetControllerReference(graph, desired, r.Scheme)
 		if err != nil {
 			return reconcile.Result{}, err