Skip to content

Commit 1362b14

Browse files
nirrozenbaumnirrozenbaum
authored
cleanup after config api PR was merged (#1012)
* cleanup after config api pr was merged Signed-off-by: Nir Rozenbaum <[email protected]> * minor changes Signed-off-by: Nir Rozenbaum <[email protected]> * typo fix Signed-off-by: Nir Rozenbaum <[email protected]> * add comment for how to register out of tree plugin Signed-off-by: Nir Rozenbaum <[email protected]> --------- Signed-off-by: Nir Rozenbaum <[email protected]>
1 parent e29fa4b commit 1362b14

19 files changed

+111
-130
lines changed

‎cmd/epp/main.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ import (
2626

2727
func main() {
2828
// Register all known plugin factories
29-
runner.RegisterAllPlgugins()
29+
runner.RegisterAllPlugins()
30+
// For adding out-of-tree plugins to the plugins registry, use the following:
31+
// plugins.Register(my-out-of-tree-plugin-name, my-out-of-tree-plugin-factory-function)
3032

3133
if err := runner.NewRunner().Run(ctrl.SetupSignalHandler()); err != nil {
3234
os.Exit(1)

‎cmd/epp/runner/register.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ import (
2525
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/scorer"
2626
)
2727

28-
// RegisterAllPlgugins registers the factory functions of all known plugins
29-
func RegisterAllPlgugins() {
28+
// RegisterAllPlugins registers the factory functions of all known plugins
29+
func RegisterAllPlugins() {
3030
plugins.Register(filter.LeastKVCacheFilterName, filter.LeastKVCacheFilterFactory)
3131
plugins.Register(filter.LeastQueueFilterName, filter.LeastQueueFilterFactory)
3232
plugins.Register(filter.LoraAffinityFilterName, filter.LoraAffinityFilterFactory)

‎cmd/epp/runner/runner.go

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,15 +35,13 @@ import (
3535
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
3636
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
3737

38-
"sigs.k8s.io/gateway-api-inference-extension/api/config/v1alpha1"
3938
conformance_epp "sigs.k8s.io/gateway-api-inference-extension/conformance/testing-epp"
4039
"sigs.k8s.io/gateway-api-inference-extension/internal/runnable"
4140
backendmetrics "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics"
4241
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/common/config"
4342
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datastore"
4443
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics"
4544
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics/collectors"
46-
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
4745
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/requestcontrol"
4846
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/saturationdetector"
4947
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling"
@@ -217,28 +215,30 @@ func (r *Runner) Run(ctx context.Context) error {
217215
return err
218216
}
219217

220-
var theConfig *v1alpha1.EndpointPickerConfig
221-
var instantiatedPlugins map[string]plugins.Plugin
222-
223218
if len(*configText) != 0 || len(*configFile) != 0 {
224-
theConfig, err = config.LoadConfig([]byte(*configText), *configFile)
219+
theConfig, err := config.LoadConfig([]byte(*configText), *configFile)
225220
if err != nil {
226221
setupLog.Error(err, "Failed to load the configuration")
227222
return err
228223
}
229224

230225
epp := eppHandle{}
231-
instantiatedPlugins, err = config.LoadPluginReferences(theConfig.Plugins, epp)
226+
instantiatedPlugins, err := config.LoadPluginReferences(theConfig.Plugins, epp)
232227
if err != nil {
233228
setupLog.Error(err, "Failed to instantiate the plugins")
234229
return err
235230
}
236231

237-
r.schedulerConfig, err = scheduling.LoadSchedulerConfig(theConfig.SchedulingProfiles, instantiatedPlugins, setupLog)
232+
r.schedulerConfig, err = scheduling.LoadSchedulerConfig(theConfig.SchedulingProfiles, instantiatedPlugins)
238233
if err != nil {
239234
setupLog.Error(err, "Failed to create Scheduler configuration")
240235
return err
241236
}
237+
238+
// Add requestcontrol plugins
239+
if instantiatedPlugins != nil {
240+
r.requestControlConfig = requestcontrol.LoadRequestControlConfig(instantiatedPlugins)
241+
}
242242
}
243243

244244
// --- Initialize Core EPP Components ---
@@ -250,10 +250,6 @@ func (r *Runner) Run(ctx context.Context) error {
250250

251251
saturationDetector := saturationdetector.NewDetector(sdConfig, datastore, ctrl.Log)
252252

253-
// Add requestControl plugins
254-
if instantiatedPlugins != nil {
255-
r.requestControlConfig.AddPlugins(instantiatedPlugins)
256-
}
257253
director := requestcontrol.NewDirectorWithConfig(datastore, scheduler, saturationDetector, r.requestControlConfig)
258254

259255
// --- Setup ExtProc Server Runner ---

‎pkg/epp/registry/registry.go

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎pkg/epp/requestcontrol/request_control_config.go

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@ limitations under the License.
1616

1717
package requestcontrol
1818

19-
import "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
19+
import (
20+
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
21+
)
2022

2123
// NewConfig creates a new Config object and returns its pointer.
2224
func NewConfig() *Config {
@@ -46,10 +48,22 @@ func (c *Config) WithPostResponsePlugins(plugins ...PostResponse) *Config {
4648
return c
4749
}
4850

49-
func (c *Config) AddPlugins(instances map[string]plugins.Plugin) {
50-
for _, plugin := range instances {
51-
if postResponse, ok := plugin.(PostResponse); ok {
52-
c.postResponsePlugins = append(c.postResponsePlugins, postResponse)
51+
func (c *Config) AddPlugins(pluginObjects ...plugins.Plugin) {
52+
for _, plugin := range pluginObjects {
53+
if preRequestPlugin, ok := plugin.(PreRequest); ok {
54+
c.preRequestPlugins = append(c.preRequestPlugins, preRequestPlugin)
55+
}
56+
if postResponsePlugin, ok := plugin.(PostResponse); ok {
57+
c.postResponsePlugins = append(c.postResponsePlugins, postResponsePlugin)
5358
}
5459
}
5560
}
61+
62+
func LoadRequestControlConfig(instantiatedPlugins map[string]plugins.Plugin) *Config {
63+
config := NewConfig()
64+
for _, plugin := range instantiatedPlugins {
65+
config.AddPlugins(plugin)
66+
}
67+
68+
return config
69+
}

‎pkg/epp/scheduling/framework/plugins/filter/filter_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ func TestLoRASoftAffinityDistribution(t *testing.T) {
203203
expectedAvailabilityPercent := 100 - expectedAffinityPercent
204204

205205
// initialize LoraAffinityFilter
206-
LoraAffinityFilter := NewLoraAffinityFilter()
206+
LoraAffinityFilter := NewLoraAffinityFilter(config.Conf.LoraAffinityThreshold)
207207

208208
for range numIterations {
209209
result := LoraAffinityFilter.Filter(context.Background(), req, types.NewCycleState(), pods)

‎pkg/epp/scheduling/framework/plugins/filter/least_kvcache_filter.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,14 @@ import (
2626
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
2727
)
2828

29-
const LeastKVCacheFilterName = "least-KV-cache"
29+
const (
30+
LeastKVCacheFilterName = "least-KV-cache"
31+
)
3032

3133
// compile-time type validation
3234
var _ framework.Filter = &LeastKVCacheFilter{}
3335

34-
// LeastKVCacheFilterFactory is the plugin factory function for the Least KV Cache filter
36+
// LeastKVCacheFilterFactory defines the factory function for LeastKVCacheFilter.
3537
func LeastKVCacheFilterFactory(name string, _ json.RawMessage, _ plugins.Handle) (plugins.Plugin, error) {
3638
return NewLeastKVCacheFilter(), nil
3739
}

‎pkg/epp/scheduling/framework/plugins/filter/least_queue_filter.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,14 @@ import (
2626
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
2727
)
2828

29-
const LeastQueueFilterName = "least-queue"
29+
const (
30+
LeastQueueFilterName = "least-queue"
31+
)
3032

3133
// compile-time type validation
3234
var _ framework.Filter = &LeastQueueFilter{}
3335

34-
// LeastQueueFilterFactory is the plugin factory function for the Least Queue filter
36+
// LeastQueueFilterFactory defines the factory function for LeastQueueFilter.
3537
func LeastQueueFilterFactory(name string, _ json.RawMessage, _ plugins.Handle) (plugins.Plugin, error) {
3638
return NewLeastQueueFilter(), nil
3739
}

‎pkg/epp/scheduling/framework/plugins/filter/lora_affinity_filter.go

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ import (
2929
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
3030
)
3131

32-
const LoraAffinityFilterName = "lora-affinity"
32+
const (
33+
LoraAffinityFilterName = "lora-affinity"
34+
)
3335

3436
type loraAffinityFilterParameters struct {
3537
Threshold float64 `json:"threshold"`
@@ -38,19 +40,19 @@ type loraAffinityFilterParameters struct {
3840
// compile-time type validation
3941
var _ framework.Filter = &LoraAffinityFilter{}
4042

41-
// LoraAffinityFilterFactory is the factory function for the LoraAffinity filter
43+
// LoraAffinityFilterFactory defines the factory function for LoraAffinityFilter.
4244
func LoraAffinityFilterFactory(name string, rawParameters json.RawMessage, _ plugins.Handle) (plugins.Plugin, error) {
4345
parameters := loraAffinityFilterParameters{Threshold: config.DefaultLoraAffinityThreshold}
4446
if err := json.Unmarshal(rawParameters, &parameters); err != nil {
45-
return nil, fmt.Errorf("failed to parse the parameters of the %s filter. Error: %s", LoraAffinityFilterName, err)
47+
return nil, fmt.Errorf("failed to parse the parameters of the '%s' filter - %w", LoraAffinityFilterName, err)
4648
}
47-
return &LoraAffinityFilter{loraAffinityThreshold: parameters.Threshold}, nil
49+
return NewLoraAffinityFilter(parameters.Threshold), nil
4850
}
4951

5052
// NewLoraAffinityFilter initializes a new LoraAffinityFilter and returns its pointer.
51-
func NewLoraAffinityFilter() *LoraAffinityFilter {
53+
func NewLoraAffinityFilter(threshold float64) *LoraAffinityFilter {
5254
return &LoraAffinityFilter{
53-
loraAffinityThreshold: config.Conf.LoraAffinityThreshold,
55+
loraAffinityThreshold: threshold,
5456
}
5557
}
5658

‎pkg/epp/scheduling/framework/plugins/filter/low_queue_filter.go

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ import (
2828
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
2929
)
3030

31-
const LowQueueFilterName = "low-queue"
31+
const (
32+
LowQueueFilterName = "low-queue"
33+
)
3234

3335
type lowQueueFilterParameters struct {
3436
Threshold int `json:"threshold"`
@@ -37,20 +39,20 @@ type lowQueueFilterParameters struct {
3739
// compile-time type validation
3840
var _ framework.Filter = &LowQueueFilter{}
3941

40-
// LowQueueFilterFactory is the factory function for the LowQueue filter
42+
// LowQueueFilterFactory defines the factory function for LowQueueFilter.
4143
func LowQueueFilterFactory(name string, rawParameters json.RawMessage, _ plugins.Handle) (plugins.Plugin, error) {
4244
parameters := lowQueueFilterParameters{Threshold: config.DefaultQueueingThresholdLoRA}
4345
if err := json.Unmarshal(rawParameters, &parameters); err != nil {
44-
return nil, fmt.Errorf("failed to parse the parameters of the %s filter. Error: %s", LowQueueFilterName, err)
46+
return nil, fmt.Errorf("failed to parse the parameters of the '%s' filter - %w", LowQueueFilterName, err)
4547
}
4648

47-
return &LowQueueFilter{queueingThresholdLoRA: parameters.Threshold}, nil
49+
return NewLowQueueFilter(parameters.Threshold), nil
4850
}
4951

5052
// NewLowQueueFilter initializes a new LowQueueFilter and returns its pointer.
51-
func NewLowQueueFilter() *LowQueueFilter {
53+
func NewLowQueueFilter(threshold int) *LowQueueFilter {
5254
return &LowQueueFilter{
53-
queueingThresholdLoRA: config.Conf.QueueingThresholdLoRA,
55+
queueingThresholdLoRA: threshold,
5456
}
5557
}
5658

0 commit comments

Comments
 (0)