diff --git a/pkg/epp/scheduling/framework/plugins/multi/prefix/plugin.go b/pkg/epp/scheduling/framework/plugins/multi/prefix/plugin.go index 4e0416720..7c6993ddb 100644 --- a/pkg/epp/scheduling/framework/plugins/multi/prefix/plugin.go +++ b/pkg/epp/scheduling/framework/plugins/multi/prefix/plugin.go @@ -216,7 +216,12 @@ func (p *Plugin) PreRequest(ctx context.Context, request *types.LLMRequest, sche return } - p.indexer.Add(state.PrefixHashes, ServerID(targetPod.NamespacedName)) + // This function is just adding data, it does not need to block other operations. + // TODO: look into making this entire function async, none of this needs to be done in-band + // The PR that introduces this change is meant as a cherrypick, so it was minimally invasive. + go func() { + p.indexer.Add(state.PrefixHashes, ServerID(targetPod.NamespacedName)) + }() total := len(state.PrefixHashes) matchLen := state.PrefixCacheServers[ServerID(targetPod.NamespacedName)] diff --git a/pkg/epp/scheduling/framework/plugins/picker/max_score_picker.go b/pkg/epp/scheduling/framework/plugins/picker/max_score_picker.go index a0e973fe6..90100d7e6 100644 --- a/pkg/epp/scheduling/framework/plugins/picker/max_score_picker.go +++ b/pkg/epp/scheduling/framework/plugins/picker/max_score_picker.go @@ -82,8 +82,8 @@ func (p *MaxScorePicker) TypedName() plugins.TypedName { // Pick selects the pod with the maximum score from the list of candidates. func (p *MaxScorePicker) Pick(ctx context.Context, cycleState *types.CycleState, scoredPods []*types.ScoredPod) *types.ProfileRunResult { - log.FromContext(ctx).V(logutil.DEBUG).Info(fmt.Sprintf("Selecting maximum '%d' pods from %d candidates sorted by max score: %+v", p.maxNumOfEndpoints, - len(scoredPods), scoredPods)) + log.FromContext(ctx).V(logutil.DEBUG).Info("Selecting pods from candidates sorted by max score: ", "NumberOfPods", p.maxNumOfEndpoints, + "scoredPodsLength", len(scoredPods), "scoredPods", scoredPods) // TODO: merge this with the logic in RandomPicker // Rand package is not safe for concurrent use, so we create a new instance. diff --git a/pkg/epp/scheduling/framework/scheduler_profile.go b/pkg/epp/scheduling/framework/scheduler_profile.go index 9790b9843..9c0ae0abc 100644 --- a/pkg/epp/scheduling/framework/scheduler_profile.go +++ b/pkg/epp/scheduling/framework/scheduler_profile.go @@ -181,7 +181,7 @@ func (p *SchedulerProfile) runPickerPlugin(ctx context.Context, cycleState *type i++ } - loggerDebug.Info("Running picker plugin", "plugin", p.picker.TypedName(), "pods-weighted-score", fmt.Sprint(weightedScorePerPod)) + loggerDebug.Info("Running picker plugin", "plugin", p.picker.TypedName(), "pods-weighted-score", weightedScorePerPod) before := time.Now() result := p.picker.Pick(ctx, cycleState, scoredPods) metrics.RecordPluginProcessingLatency(PickerExtensionPoint, p.picker.TypedName().Type, p.picker.TypedName().Name, time.Since(before))