diff --git a/api/v1/inferencepool_types.go b/api/v1/inferencepool_types.go
index 98f45d5bb..145c75da9 100644
--- a/api/v1/inferencepool_types.go
+++ b/api/v1/inferencepool_types.go
@@ -69,14 +69,6 @@ type InferencePoolSpec struct {
 	// +kubebuilder:validation:Required
 	TargetPortNumber int32 `json:"targetPortNumber"`
 
-	// EndpointPickerConfig specifies the configuration needed by the proxy to discover and connect to the endpoint
-	// picker service that picks endpoints for the requests routed to this pool.
-	EndpointPickerConfig `json:",inline"`
-}
-
-// EndpointPickerConfig specifies the configuration needed by the proxy to discover and connect to the endpoint picker extension.
-// This type is intended to be a union of mutually exclusive configuration options that we may add in the future.
-type EndpointPickerConfig struct {
 	// Extension configures an endpoint picker as an extension service.
 	//
 	// +kubebuilder:validation:Required
diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
index adf19acbe..b4fc32a0a 100644
--- a/api/v1/zz_generated.deepcopy.go
+++ b/api/v1/zz_generated.deepcopy.go
@@ -25,26 +25,6 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *EndpointPickerConfig) DeepCopyInto(out *EndpointPickerConfig) {
-	*out = *in
-	if in.ExtensionRef != nil {
-		in, out := &in.ExtensionRef, &out.ExtensionRef
-		*out = new(Extension)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointPickerConfig.
-func (in *EndpointPickerConfig) DeepCopy() *EndpointPickerConfig {
-	if in == nil {
-		return nil
-	}
-	out := new(EndpointPickerConfig)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Extension) DeepCopyInto(out *Extension) {
 	*out = *in
@@ -181,7 +161,11 @@ func (in *InferencePoolSpec) DeepCopyInto(out *InferencePoolSpec) {
 			(*out)[key] = val
 		}
 	}
-	in.EndpointPickerConfig.DeepCopyInto(&out.EndpointPickerConfig)
+	if in.ExtensionRef != nil {
+		in, out := &in.ExtensionRef, &out.ExtensionRef
+		*out = new(Extension)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferencePoolSpec.
diff --git a/client-go/applyconfiguration/api/v1/endpointpickerconfig.go b/client-go/applyconfiguration/api/v1/endpointpickerconfig.go
deleted file mode 100644
index 9877d40d9..000000000
--- a/client-go/applyconfiguration/api/v1/endpointpickerconfig.go
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by applyconfiguration-gen. DO NOT EDIT.
-
-package v1
-
-// EndpointPickerConfigApplyConfiguration represents a declarative configuration of the EndpointPickerConfig type for use
-// with apply.
-type EndpointPickerConfigApplyConfiguration struct {
-	ExtensionRef *ExtensionApplyConfiguration `json:"extensionRef,omitempty"`
-}
-
-// EndpointPickerConfigApplyConfiguration constructs a declarative configuration of the EndpointPickerConfig type for use with
-// apply.
-func EndpointPickerConfig() *EndpointPickerConfigApplyConfiguration {
-	return &EndpointPickerConfigApplyConfiguration{}
-}
-
-// WithExtensionRef sets the ExtensionRef field in the declarative configuration to the given value
-// and returns the receiver, so that objects can be built by chaining "With" function invocations.
-// If called multiple times, the ExtensionRef field is set to the value of the last call.
-func (b *EndpointPickerConfigApplyConfiguration) WithExtensionRef(value *ExtensionApplyConfiguration) *EndpointPickerConfigApplyConfiguration {
-	b.ExtensionRef = value
-	return b
-}
diff --git a/client-go/applyconfiguration/api/v1/inferencepoolspec.go b/client-go/applyconfiguration/api/v1/inferencepoolspec.go
index d9c08b238..4831a4434 100644
--- a/client-go/applyconfiguration/api/v1/inferencepoolspec.go
+++ b/client-go/applyconfiguration/api/v1/inferencepoolspec.go
@@ -25,9 +25,9 @@ import (
 // InferencePoolSpecApplyConfiguration represents a declarative configuration of the InferencePoolSpec type for use
 // with apply.
 type InferencePoolSpecApplyConfiguration struct {
-	Selector                               map[apiv1.LabelKey]apiv1.LabelValue `json:"selector,omitempty"`
-	TargetPortNumber                       *int32                              `json:"targetPortNumber,omitempty"`
-	EndpointPickerConfigApplyConfiguration `json:",inline"`
+	Selector         map[apiv1.LabelKey]apiv1.LabelValue `json:"selector,omitempty"`
+	TargetPortNumber *int32                              `json:"targetPortNumber,omitempty"`
+	ExtensionRef     *ExtensionApplyConfiguration        `json:"extensionRef,omitempty"`
 }
 
 // InferencePoolSpecApplyConfiguration constructs a declarative configuration of the InferencePoolSpec type for use with
@@ -62,6 +62,6 @@ func (b *InferencePoolSpecApplyConfiguration) WithTargetPortNumber(value int32)
 // and returns the receiver, so that objects can be built by chaining "With" function invocations.
 // If called multiple times, the ExtensionRef field is set to the value of the last call.
 func (b *InferencePoolSpecApplyConfiguration) WithExtensionRef(value *ExtensionApplyConfiguration) *InferencePoolSpecApplyConfiguration {
-	b.EndpointPickerConfigApplyConfiguration.ExtensionRef = value
+	b.ExtensionRef = value
 	return b
 }
diff --git a/client-go/applyconfiguration/utils.go b/client-go/applyconfiguration/utils.go
index 46895d8f0..d49e54744 100644
--- a/client-go/applyconfiguration/utils.go
+++ b/client-go/applyconfiguration/utils.go
@@ -34,8 +34,6 @@ import (
 func ForKind(kind schema.GroupVersionKind) interface{} {
 	switch kind {
 	// Group=inference.networking.k8s.io, Version=v1
-	case v1.SchemeGroupVersion.WithKind("EndpointPickerConfig"):
-		return &apiv1.EndpointPickerConfigApplyConfiguration{}
 	case v1.SchemeGroupVersion.WithKind("Extension"):
 		return &apiv1.ExtensionApplyConfiguration{}
 	case v1.SchemeGroupVersion.WithKind("ExtensionConnection"):