Merge branch 'kubernetes-sigs:main' into update-helm-chart

Author: rahulgurnani

Dockerfile

@@ -18,6 +18,7 @@ RUN go mod download
 
 # Sources
 COPY cmd/epp ./cmd/epp
+COPY pkg/common ./pkg/common
 COPY pkg/epp ./pkg/epp
 COPY internal ./internal
 COPY apix ./apix

Makefile

@@ -24,7 +24,6 @@ SHELL = /usr/bin/env bash -o pipefail
 GIT_COMMIT_SHA ?= "$(shell git rev-parse HEAD 2>/dev/null)"
 GIT_TAG ?= $(shell git describe --tags --dirty --always)
 PLATFORMS ?= linux/amd64
-PUBLISH_PLATFORMS ?= linux/amd64,linux/arm64
 DOCKER_BUILDX_CMD ?= docker buildx
 IMAGE_BUILD_CMD ?= $(DOCKER_BUILDX_CMD) build
 IMAGE_BUILD_EXTRA_OPTS ?=
@@ -74,11 +73,6 @@ SYNCER_IMAGE_BUILD_EXTRA_OPTS += -t $(SYNCER_IMAGE_EXTRA_TAG)
 BBR_IMAGE_BUILD_EXTRA_OPTS += -t $(BBR_IMAGE_EXTRA_TAG)
 endif
 
-# Allow `make MULTI=true ...` in CI to switch to multi-arch.
-ifdef MULTI
-  PLATFORMS := $(PUBLISH_PLATFORMS)
-endif
-
 
 # The name of the kind cluster to use for the "kind-load" target.
 KIND_CLUSTER ?= kind

apix/config/v1alpha1/defaults.go

@@ -18,11 +18,11 @@ package v1alpha1
 
 import (
 	"encoding/json"
+	"fmt"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// +k8s:defaulter-gen=true
 // +kubebuilder:object:root=true
 
 // EndpointPickerConfig is the Schema for the endpointpickerconfigs API
@@ -41,6 +41,14 @@ type EndpointPickerConfig struct {
 	SchedulingProfiles []SchedulingProfile `json:"schedulingProfiles"`
 }
 
+func (cfg EndpointPickerConfig) String() string {
+	return fmt.Sprintf(
+		"{Plugins: %v, SchedulingProfiles: %v}",
+		cfg.Plugins,
+		cfg.SchedulingProfiles,
+	)
+}
+
 // PluginSpec contains the information that describes a plugin that
 // will be instantiated.
 type PluginSpec struct {
@@ -61,6 +69,14 @@ type PluginSpec struct {
 	Parameters json.RawMessage `json:"parameters"`
 }
 
+func (ps PluginSpec) String() string {
+	var parameters string
+	if ps.Parameters != nil {
+		parameters = fmt.Sprintf(", Parameters: %s", ps.Parameters)
+	}
+	return fmt.Sprintf("{%s/%s%s}", ps.Name, ps.Type, parameters)
+}
+
 // SchedulingProfile contains the information to create a SchedulingProfile
 // entry to be used by the scheduler.
 type SchedulingProfile struct {
@@ -75,6 +91,10 @@ type SchedulingProfile struct {
 	Plugins []SchedulingPlugin `json:"plugins"`
 }
 
+func (sp SchedulingProfile) String() string {
+	return fmt.Sprintf("{Name: %s, Plugins: %v}", sp.Name, sp.Plugins)
+}
+
 // SchedulingPlugin describes a plugin that will be associated with a
 // SchedulingProfile entry.
 type SchedulingPlugin struct {
@@ -90,3 +110,11 @@ type SchedulingPlugin struct {
 	// Weight is the weight fo be used if this plugin is a Scorer.
 	Weight *int `json:"weight"`
 }
+
+func (sp SchedulingPlugin) String() string {
+	var weight string
+	if sp.Weight != nil {
+		weight = fmt.Sprintf(", Weight: %d", *sp.Weight)
+	}
+	return fmt.Sprintf("{PluginRef: %s%s}", sp.PluginRef, weight)
+}

apix/config/v1alpha1/endpointpickerconfig_types.go

@@ -63,20 +63,6 @@ type InferenceObjectiveList struct {
 // creation timestamp, will be selected to remain valid. In the event of a race
 // condition, one will be selected at random.
 type InferenceObjectiveSpec struct {
-	// ModelName is the name of the model as it will be set in the "model" parameter for an incoming request.
-	// ModelNames must be unique for a referencing InferencePool
-	// (names can be reused for a different pool in the same cluster).
-	// The modelName with the oldest creation timestamp is retained, and the incoming
-	// InferenceObjective's Ready status is set to false with a corresponding reason.
-	// In the rare case of a race condition, one Model will be selected randomly to be considered valid, and the other rejected.
-	// Names can be reserved without an underlying model configured in the pool.
-	// This can be done by specifying a target model and setting the weight to zero,
-	// an error will be returned specifying that no valid target model is found.
-	//
-	// +kubebuilder:validation:MaxLength=256
-	// +kubebuilder:validation:Required
-	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="modelName is immutable"
-	ModelName string `json:"modelName"`
 
 	// Criticality defines how important it is to serve the model compared to other models referencing the same pool.
 	// Criticality impacts how traffic is handled in resource constrained situations. It handles this by

apix/config/v1alpha1/zz_generated.defaults.go

@@ -13,7 +13,6 @@ steps:
     - EXTRA_TAG=$_PULL_BASE_REF
     - DOCKER_BUILDX_CMD=/buildx-entrypoint
     - GIT_COMMIT_SHA=$_PULL_BASE_SHA
-    - MULTI=true
   - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20240718-5ef92b5c36
     entrypoint: make
     args:

apix/v1alpha2/inferenceobjective_types.go

@@ -32,6 +32,7 @@ import (
 	"go.uber.org/zap/zapcore"
 	"google.golang.org/grpc"
 	healthPb "google.golang.org/grpc/health/grpc_health_v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -41,6 +42,7 @@ import (
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"sigs.k8s.io/gateway-api-inference-extension/internal/runnable"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/common"
 	backendmetrics "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/config/loader"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datastore"
@@ -50,7 +52,6 @@ import (
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/requestcontrol"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/saturationdetector"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling"
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/filter"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/multi/prefix"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/picker"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/profile"
@@ -82,6 +83,10 @@ var (
 		"pool-name",
 		runserver.DefaultPoolName,
 		"Name of the InferencePool this Endpoint Picker is associated with.")
+	poolGroup = flag.String(
+		"pool-group",
+		runserver.DefaultPoolGroup,
+		"group of the InferencePool this Endpoint Picker is associated with.")
 	poolNamespace = flag.String(
 		"pool-namespace",
 		runserver.DefaultPoolNamespace,
@@ -104,20 +109,6 @@ var (
 		"The path to the certificate for secure serving. The certificate and private key files "+
 			"are assumed to be named tls.crt and tls.key, respectively. If not set, and secureServing is enabled, "+
 			"then a self-signed certificate is used.")
-	// header/metadata flags
-	destinationEndpointHintKey = flag.String(
-		"destination-endpoint-hint-key",
-		runserver.DefaultDestinationEndpointHintKey,
-		"Header and response metadata key used by Envoy to route to the appropriate pod. This must match Envoy configuration.")
-	destinationEndpointHintMetadataNamespace = flag.String(
-		"destination-endpoint-hint-metadata-namespace",
-		runserver.DefaultDestinationEndpointHintMetadataNamespace,
-		"The key for the outer namespace struct in the metadata field of the extproc response that is used to wrap the"+
-			"target endpoint. If not set, then an outer namespace struct should not be created.")
-	fairnessIDHeaderKey = flag.String(
-		"fairness-id-header-key",
-		runserver.DefaultFairnessIDHeaderKey,
-		"The header key used to pass the fairness ID to be used in Flow Control.")
 	// metric flags
 	totalQueuedRequestsMetric = flag.String(
 		"total-queued-requests-metric",
@@ -196,7 +187,6 @@ func bindEnvToFlags() {
 		"MODEL_SERVER_METRICS_PATH":                       "model-server-metrics-path",
 		"MODEL_SERVER_METRICS_SCHEME":                     "model-server-metrics-scheme",
 		"MODEL_SERVER_METRICS_HTTPS_INSECURE_SKIP_VERIFY": "model-server-metrics-https-insecure-skip-verify",
-		"DESTINATION_ENDPOINT_HINT_KEY":                   "destination-endpoint-hint-key",
 		"POOL_NAME":                                       "pool-name",
 		"POOL_NAMESPACE":                                  "pool-namespace",
 		// durations & bools work too; flag.Set expects the *string* form
@@ -301,7 +291,15 @@ func (r *Runner) Run(ctx context.Context) error {
 		Name:      *poolName,
 		Namespace: *poolNamespace,
 	}
-	mgr, err := runserver.NewDefaultManager(poolNamespacedName, cfg, metricsServerOptions)
+	poolGroupKind := schema.GroupKind{
+		Group: *poolGroup,
+		Kind:  "InferencePool",
+	}
+	poolGKNN := common.GKNN{
+		NamespacedName: poolNamespacedName,
+		GroupKind:      poolGroupKind,
+	}
+	mgr, err := runserver.NewDefaultManager(poolGKNN, cfg, metricsServerOptions)
 	if err != nil {
 		setupLog.Error(err, "Failed to create controller manager")
 		return err
@@ -339,19 +337,17 @@ func (r *Runner) Run(ctx context.Context) error {
 
 	// --- Setup ExtProc Server Runner ---
 	serverRunner := &runserver.ExtProcServerRunner{
-		GrpcPort:                                 *grpcPort,
-		DestinationEndpointHintMetadataNamespace: *destinationEndpointHintMetadataNamespace,
-		DestinationEndpointHintKey:               *destinationEndpointHintKey,
-		FairnessIDHeaderKey:                      *fairnessIDHeaderKey,
-		PoolNamespacedName:                       poolNamespacedName,
-		Datastore:                                datastore,
-		SecureServing:                            *secureServing,
-		HealthChecking:                           *healthChecking,
-		CertPath:                                 *certPath,
-		RefreshPrometheusMetricsInterval:         *refreshPrometheusMetricsInterval,
-		MetricsStalenessThreshold:                *metricsStalenessThreshold,
-		Director:                                 director,
-		SaturationDetector:                       saturationDetector,
+		GrpcPort:                         *grpcPort,
+		PoolNamespacedName:               poolNamespacedName,
+		PoolGKNN:                         poolGKNN,
+		Datastore:                        datastore,
+		SecureServing:                    *secureServing,
+		HealthChecking:                   *healthChecking,
+		CertPath:                         *certPath,
+		RefreshPrometheusMetricsInterval: *refreshPrometheusMetricsInterval,
+		MetricsStalenessThreshold:        *metricsStalenessThreshold,
+		Director:                         director,
+		SaturationDetector:               saturationDetector,
 	}
 	if err := serverRunner.SetupWithManager(ctx, mgr); err != nil {
 		setupLog.Error(err, "Failed to setup EPP controllers")
@@ -382,11 +378,6 @@ func (r *Runner) Run(ctx context.Context) error {
 
 // registerInTreePlugins registers the factory functions of all known plugins
 func (r *Runner) registerInTreePlugins() {
-	plugins.Register(filter.DecisionTreeFilterType, filter.DecisionTreeFilterFactory)
-	plugins.Register(filter.LeastKVCacheFilterType, filter.LeastKVCacheFilterFactory)
-	plugins.Register(filter.LeastQueueFilterType, filter.LeastQueueFilterFactory)
-	plugins.Register(filter.LoraAffinityFilterType, filter.LoraAffinityFilterFactory)
-	plugins.Register(filter.LowQueueFilterType, filter.LowQueueFilterFactory)
 	plugins.Register(prefix.PrefixCachePluginType, prefix.PrefixCachePluginFactory)
 	plugins.Register(picker.MaxScorePickerType, picker.MaxScorePickerFactory)
 	plugins.Register(picker.RandomPickerType, picker.RandomPickerFactory)
@@ -403,6 +394,8 @@ func (r *Runner) parsePluginsConfiguration(ctx context.Context) error {
 		return nil // configuring through code, not through file
 	}
 
+	logger := log.FromContext(ctx)
+
 	var configBytes []byte
 	if *configText != "" {
 		configBytes = []byte(*configText)
@@ -416,20 +409,17 @@ func (r *Runner) parsePluginsConfiguration(ctx context.Context) error {
 
 	r.registerInTreePlugins()
 	handle := plugins.NewEppHandle(ctx)
-	config, err := loader.LoadConfig(configBytes, handle)
+	config, err := loader.LoadConfig(configBytes, handle, logger)
 	if err != nil {
 		return fmt.Errorf("failed to load the configuration - %w", err)
 	}
 
-	r.schedulerConfig, err = loader.LoadSchedulerConfig(config.SchedulingProfiles, handle)
-	if err != nil {
-		return fmt.Errorf("failed to create Scheduler configuration - %w", err)
-	}
+	r.schedulerConfig = config.SchedulerConfig
 
 	// Add requestControl plugins
 	r.requestControlConfig.AddPlugins(handle.GetAllPlugins()...)
 
-	log.FromContext(ctx).Info("loaded configuration from file/text successfully")
+	logger.Info("loaded configuration from file/text successfully")
 	return nil
 }