Changes InferencePool EPP Flags (#152)

Signed-off-by: Daneyon Hansen <[email protected]>

Author: danehans

pkg/ext-proc/backend/endpointslice_reconciler.go

@@ -21,13 +21,13 @@ var (
 
 type EndpointSliceReconciler struct {
 	client.Client
-	Scheme         *runtime.Scheme
-	Record         record.EventRecorder
-	ServerPoolName string
-	ServiceName    string
-	Zone           string
-	Namespace      string
-	Datastore      *K8sDatastore
+	Scheme      *runtime.Scheme
+	Record      record.EventRecorder
+	PoolName    string
+	ServiceName string
+	Zone        string
+	Namespace   string
+	Datastore   *K8sDatastore
 }
 
 func (c *EndpointSliceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {

pkg/ext-proc/backend/inferencemodel_reconciler.go

@@ -13,15 +13,15 @@ import (
 
 type InferenceModelReconciler struct {
 	client.Client
-	Scheme         *runtime.Scheme
-	Record         record.EventRecorder
-	Datastore      *K8sDatastore
-	ServerPoolName string
-	Namespace      string
+	Scheme        *runtime.Scheme
+	Record        record.EventRecorder
+	Datastore     *K8sDatastore
+	PoolName      string
+	PoolNamespace string
 }
 
 func (c *InferenceModelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	if req.Namespace != c.Namespace {
+	if req.Namespace != c.PoolNamespace {
 		return ctrl.Result{}, nil
 	}
 	klog.V(1).Info("reconciling InferenceModel", req.NamespacedName)
@@ -43,8 +43,8 @@ func (c *InferenceModelReconciler) SetupWithManager(mgr ctrl.Manager) error {
 }
 
 func (c *InferenceModelReconciler) updateDatastore(infModel *v1alpha1.InferenceModel) {
-	if infModel.Spec.PoolRef.Name == c.ServerPoolName {
-		klog.V(1).Infof("Incoming pool ref %v, server pool name: %v", infModel.Spec.PoolRef, c.ServerPoolName)
+	if infModel.Spec.PoolRef.Name == c.PoolName {
+		klog.V(1).Infof("Incoming pool ref %v, server pool name: %v", infModel.Spec.PoolRef, c.PoolName)
 		klog.V(1).Infof("Adding/Updating inference model: %v", infModel.Spec.ModelName)
 		c.Datastore.InferenceModels.Store(infModel.Spec.ModelName, infModel)
 		return

pkg/ext-proc/backend/inferencemodel_reconciler_test.go

@@ -125,8 +125,8 @@ func TestUpdateDatastore_InferenceModelReconciler(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			InferenceModelReconciler := &InferenceModelReconciler{
-				Datastore:      test.datastore,
-				ServerPoolName: test.datastore.inferencePool.Name,
+				Datastore: test.datastore,
+				PoolName:  test.datastore.inferencePool.Name,
 			}
 			InferenceModelReconciler.updateDatastore(test.incomingService)
 

pkg/ext-proc/backend/inferencepool_reconciler.go

@@ -16,16 +16,16 @@ import (
 // will have the proper controller that will create/manage objects on behalf of the server pool.
 type InferencePoolReconciler struct {
 	client.Client
-	Scheme         *runtime.Scheme
-	Record         record.EventRecorder
-	ServerPoolName string
-	Namespace      string
-	Datastore      *K8sDatastore
-	Zone           string
+	Scheme        *runtime.Scheme
+	Record        record.EventRecorder
+	PoolName      string
+	PoolNamespace string
+	Datastore     *K8sDatastore
+	Zone          string
 }
 
 func (c *InferencePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	if req.NamespacedName.Name != c.ServerPoolName || req.NamespacedName.Namespace != c.Namespace {
+	if req.NamespacedName.Name != c.PoolName || req.NamespacedName.Namespace != c.PoolNamespace {
 		return ctrl.Result{}, nil
 	}
 	klog.V(1).Info("reconciling InferencePool", req.NamespacedName)

pkg/ext-proc/main.go

@@ -36,18 +36,18 @@ var (
 		"targetPodHeader",
 		"target-pod",
 		"Header key used by Envoy to route to the appropriate pod. This must match Envoy configuration.")
-	serverPoolName = flag.String(
-		"serverPoolName",
+	poolName = flag.String(
+		"poolName",
 		"",
-		"Name of the serverPool this Endpoint Picker is associated with.")
+		"Name of the InferencePool this Endpoint Picker is associated with.")
+	poolNamespace = flag.String(
+		"poolNamespace",
+		"default",
+		"Namespace of the InferencePool this Endpoint Picker is associated with.")
 	serviceName = flag.String(
 		"serviceName",
 		"",
-		"Name of the service that will be used to read the endpointslices from")
-	namespace = flag.String(
-		"namespace",
-		"default",
-		"The Namespace that the server pool should exist in.")
+		"Name of the Service that will be used to read EndpointSlices from")
 	zone = flag.String(
 		"zone",
 		"",
@@ -114,35 +114,35 @@ func main() {
 	}
 
 	if err := (&backend.InferencePoolReconciler{
-		Datastore:      datastore,
-		Scheme:         mgr.GetScheme(),
-		Client:         mgr.GetClient(),
-		ServerPoolName: *serverPoolName,
-		Namespace:      *namespace,
-		Record:         mgr.GetEventRecorderFor("InferencePool"),
+		Datastore:     datastore,
+		Scheme:        mgr.GetScheme(),
+		Client:        mgr.GetClient(),
+		PoolName:      *poolName,
+		PoolNamespace: *poolNamespace,
+		Record:        mgr.GetEventRecorderFor("InferencePool"),
 	}).SetupWithManager(mgr); err != nil {
 		klog.Error(err, "Error setting up InferencePoolReconciler")
 	}
 
 	if err := (&backend.InferenceModelReconciler{
-		Datastore:      datastore,
-		Scheme:         mgr.GetScheme(),
-		Client:         mgr.GetClient(),
-		ServerPoolName: *serverPoolName,
-		Namespace:      *namespace,
-		Record:         mgr.GetEventRecorderFor("InferenceModel"),
+		Datastore:     datastore,
+		Scheme:        mgr.GetScheme(),
+		Client:        mgr.GetClient(),
+		PoolName:      *poolName,
+		PoolNamespace: *poolNamespace,
+		Record:        mgr.GetEventRecorderFor("InferenceModel"),
 	}).SetupWithManager(mgr); err != nil {
 		klog.Error(err, "Error setting up InferenceModelReconciler")
 	}
 
 	if err := (&backend.EndpointSliceReconciler{
-		Datastore:      datastore,
-		Scheme:         mgr.GetScheme(),
-		Client:         mgr.GetClient(),
-		Record:         mgr.GetEventRecorderFor("endpointslice"),
-		ServiceName:    *serviceName,
-		Zone:           *zone,
-		ServerPoolName: *serverPoolName,
+		Datastore:   datastore,
+		Scheme:      mgr.GetScheme(),
+		Client:      mgr.GetClient(),
+		Record:      mgr.GetEventRecorderFor("endpointslice"),
+		ServiceName: *serviceName,
+		Zone:        *zone,
+		PoolName:    *poolName,
 	}).SetupWithManager(mgr); err != nil {
 		klog.Error(err, "Error setting up EndpointSliceReconciler")
 	}

pkg/manifests/ext_proc.yaml

@@ -51,7 +51,7 @@ spec:
         # TODO(https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/34) Update the image and args.
         image: us-central1-docker.pkg.dev/k8s-staging-images/llm-instance-gateway/epp:main
         args:
-        - -serverPoolName 
+        - -poolName
         - "vllm-llama2-7b-pool"
         - -v
         - "3"