unload: only unload unused runners

We can't unload runners that are in-use, otherwise state corruption will
occur in release (which will likely result in a panic() at some point on
subsequent inference). If we want a preemption mechanism, additional
refactoring will need to occur.

Signed-off-by: Jacob Howard <[email protected]>

Author: xenoscopic

pkg/inference/scheduling/loader.go

@@ -182,7 +182,8 @@ func (l *loader) evict(idleOnly bool) int {
 func (l *loader) evictRunner(backend, model string) int {
 	allBackends := backend == ""
 	for r, slot := range l.runners {
-		if (allBackends || r.backend == backend) && r.model == model {
+		unused := l.references[slot] == 0
+		if unused && (allBackends || r.backend == backend) && r.model == model {
 			l.log.Infof("Evicting %s backend runner with model %s in %s mode",
 				r.backend, r.model, r.mode,
 			)