feat(scheduler): add endpoint to retrieve model configurations

Add GET /engines/_configure endpoint that returns active model configurations, similar to Ollama's /api/show endpoint. Accepts optional "model" query parameter to filter configs for a specific model; returns all configs if no model specified.

Signed-off-by: Dorin Geman <[email protected]>

Author: doringeman

pkg/inference/backend.go

@@ -2,7 +2,9 @@ package inference
 
 import (
 	"context"
+	"fmt"
 	"net/http"
+	"strings"
 )
 
 // BackendMode encodes the mode in which a backend should operate.
@@ -40,6 +42,22 @@ func (m BackendMode) String() string {
 	}
 }
 
+// MarshalJSON implements json.Marshaler for BackendMode.
+func (m BackendMode) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + m.String() + `"`), nil
+}
+
+// UnmarshalJSON implements json.Unmarshaler for BackendMode.
+func (m *BackendMode) UnmarshalJSON(data []byte) error {
+	s := strings.Trim(string(data), "\"")
+	mode, ok := ParseBackendMode(s)
+	if !ok {
+		return fmt.Errorf("unknown backend mode: %q", s)
+	}
+	*m = mode
+	return nil
+}
+
 // ParseBackendMode converts a string mode to BackendMode.
 // It returns the parsed mode and a boolean indicating if the mode was known.
 // For unknown modes, it returns BackendModeCompletion and false.

pkg/inference/scheduling/http_handler.go

@@ -84,6 +84,7 @@ func (h *HTTPHandler) routeHandlers() map[string]http.HandlerFunc {
 	m["POST "+inference.InferencePrefix+"/unload"] = h.Unload
 	m["POST "+inference.InferencePrefix+"/{backend}/_configure"] = h.Configure
 	m["POST "+inference.InferencePrefix+"/_configure"] = h.Configure
+	m["GET "+inference.InferencePrefix+"/_configure"] = h.GetModelConfigs
 	m["GET "+inference.InferencePrefix+"/requests"] = h.scheduler.openAIRecorder.GetRecordsHandler()
 	return m
 }
@@ -350,6 +351,31 @@ func (h *HTTPHandler) Configure(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusAccepted)
 }
 
+// GetModelConfigs returns model configurations. If a model is specified in the request body,
+// returns only configs for that model; otherwise returns all configs.
+func (h *HTTPHandler) GetModelConfigs(w http.ResponseWriter, r *http.Request) {
+	model := r.URL.Query().Get("model")
+
+	configs := h.scheduler.GetAllModelConfigs(r.Context())
+
+	if model != "" {
+		modelID := h.scheduler.modelManager.ResolveID(model)
+		filtered := configs[:0]
+		for _, entry := range configs {
+			if entry.ModelID == modelID {
+				filtered = append(filtered, entry)
+			}
+		}
+		configs = filtered
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(configs); err != nil {
+		http.Error(w, fmt.Sprintf("Failed to encode response: %v", err), http.StatusInternalServerError)
+		return
+	}
+}
+
 // ServeHTTP implements net/http.Handler.ServeHTTP.
 func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.lock.RLock()

pkg/inference/scheduling/loader.go

@@ -626,3 +626,39 @@ func (l *loader) setRunnerConfig(ctx context.Context, backendName, modelID strin
 	l.runnerConfigs[configKey] = runnerConfig
 	return nil
 }
+
+// ModelConfigEntry represents a model configuration entry with its associated metadata.
+type ModelConfigEntry struct {
+	Backend string
+	Model   string
+	ModelID string
+	Mode    inference.BackendMode
+	Config  inference.BackendConfiguration
+}
+
+// getAllRunnerConfigs retrieves all runner configurations.
+func (l *loader) getAllRunnerConfigs(ctx context.Context) []ModelConfigEntry {
+	if !l.lock(ctx) {
+		return nil
+	}
+	defer l.unlock()
+
+	entries := make([]ModelConfigEntry, 0, len(l.runnerConfigs))
+	for key, config := range l.runnerConfigs {
+		model, err := l.modelManager.GetLocal(key.modelID)
+		if err == nil {
+			modelName := ""
+			if len(model.Tags()) > 0 {
+				modelName = model.Tags()[0]
+			}
+			entries = append(entries, ModelConfigEntry{
+				Backend: key.backend,
+				Model:   modelName,
+				ModelID: key.modelID,
+				Mode:    key.mode,
+				Config:  config,
+			})
+		}
+	}
+	return entries
+}

pkg/inference/scheduling/scheduler.go

@@ -300,3 +300,8 @@ func (s *Scheduler) ConfigureRunner(ctx context.Context, backend inference.Backe
 
 	return backend, nil
 }
+
+// GetAllModelConfigs retrieves all active model configurations.
+func (s *Scheduler) GetAllModelConfigs(ctx context.Context) []ModelConfigEntry {
+	return s.loader.getAllRunnerConfigs(ctx)
+}