Allow unloading multiple models at once

Once we enable running multiple models at once, it will be useful to be
able to unload multiple at a time. In preparation for that, make the
unload request accept multiple model tags, and evict those models.

Signed-off-by: Piotr Stankiewicz <[email protected]>

Author: Piotr Stankiewicz

pkg/inference/scheduling/api.go

@@ -64,9 +64,9 @@ type DiskUsage struct {
 
 // UnloadRequest is used to specify which models to unload.
 type UnloadRequest struct {
-	All     bool   `json:"all"`
-	Backend string `json:"backend"`
-	Model   string `json:"model"`
+	All     bool     `json:"all"`
+	Backend string   `json:"backend"`
+	Models  []string `json:"models"`
 }
 
 // UnloadResponse is used to return the number of unloaded runners (backend, model).

pkg/inference/scheduling/loader.go

@@ -209,7 +209,10 @@ func (l *loader) Unload(ctx context.Context, unload UnloadRequest) int {
 		if unload.All {
 			return l.evict(false)
 		} else {
-			return l.evictRunner(unload.Backend, unload.Model)
+			for _, model := range unload.Models {
+				l.evictRunner(unload.Backend, model)
+			}
+			return len(l.runners)
 		}
 	}()
 }