llama-router: add multi-engine support with configurable spawn and endpoints

- Introduce SpawnConfig struct: command, proxy_endpoints, health_endpoint
- Replace vector<string> default_spawn with full SpawnConfig
- Support per-model spawn override (vLLM, TGI, etc. alongside llama.cpp)
- Implement prefix-based endpoint filtering (simple startswith, no wildcards)
- Health endpoint now configurable per spawn config
- Validate spawn commands and proxy endpoints before execution

Default config enables /v1/, /health, /slots, /props endpoints.
Single router can now manage heterogeneous inference backends

Author: ServeurpersoCom

tools/router/router-app.cpp

@@ -17,6 +17,19 @@ RouterApp::RouterApp(RouterConfig cfg)
 
 RouterApp::~RouterApp() { stop_all(); }
 
+SpawnConfig RouterApp::resolve_spawn_config(const ModelConfig & cfg) const {
+    return is_spawn_empty(cfg.spawn) ? config.default_spawn : cfg.spawn;
+}
+
+SpawnConfig RouterApp::get_spawn_config(const std::string & model_name) {
+    std::lock_guard<std::mutex> lock(mutex);
+    auto it = model_lookup.find(model_name);
+    if (it == model_lookup.end()) {
+        return config.default_spawn;
+    }
+    return resolve_spawn_config(it->second);
+}
+
 void RouterApp::start_auto_models() {
     for (const auto & model : config.models) {
         if (model.state == "auto") {
@@ -78,7 +91,9 @@ bool RouterApp::ensure_running(const std::string & model_name, std::string & err
     int port = next_port.fetch_add(1);
     model_ports[model_name] = port;
 
-    std::vector<std::string> command = cfg.spawn.empty() ? config.default_spawn : cfg.spawn;
+    const SpawnConfig spawn_cfg = resolve_spawn_config(cfg);
+
+    std::vector<std::string> command = spawn_cfg.command;
     command.push_back("--model");
     command.push_back(expand_user_path(cfg.path));
     command.push_back("--port");
@@ -100,7 +115,8 @@ bool RouterApp::ensure_running(const std::string & model_name, std::string & err
     last_spawned_model = model_name;
     LOG_INF("Spawned %s (group '%s') with %zu args\n", model_name.c_str(), target_group.c_str(), command.size());
 
-    if (!wait_for_backend_ready(port, ROUTER_BACKEND_READY_TIMEOUT_MS, &proc_it->second)) {
+    const std::string health_endpoint = spawn_cfg.health_endpoint.empty() ? "/health" : spawn_cfg.health_endpoint;
+    if (!wait_for_backend_ready(port, health_endpoint, ROUTER_BACKEND_READY_TIMEOUT_MS, &proc_it->second)) {
         error = "backend not ready";
         LOG_ERR("Backend for %s did not become ready on port %d within %d ms\n",
                 model_name.c_str(),

tools/router/router-app.h

@@ -17,6 +17,7 @@ class RouterApp {
     bool ensure_running(const std::string & model_name, std::string & error);
     std::string upstream_for(const std::string & model_name);
     std::string get_last_spawned_model();
+    SpawnConfig get_spawn_config(const std::string & model_name);
     void stop_all();
 
     const RouterConfig & get_config() const { return config; }
@@ -29,4 +30,6 @@ class RouterApp {
     std::unordered_map<std::string, ProcessHandle> processes;
     std::unordered_map<std::string, int> model_ports;
     std::string last_spawned_model;
+
+    SpawnConfig resolve_spawn_config(const ModelConfig & cfg) const;
 };

tools/router/router-config.cpp

@@ -60,17 +60,42 @@ static std::string detect_llama_server_binary() {
 #endif
 }
 
-const std::vector<std::string> & get_default_spawn() {
-    static const std::vector<std::string> spawn = [] {
-        std::vector<std::string> default_spawn = {
-            "llama-server", "--ctx-size", "4096", "--n-gpu-layers", "99",
+static SpawnConfig parse_spawn_config(const json & data) {
+    SpawnConfig spawn;
+    if (data.contains("command")) {
+        spawn.command = data["command"].get<std::vector<std::string>>();
+    }
+    if (data.contains("proxy_endpoints")) {
+        spawn.proxy_endpoints = data["proxy_endpoints"].get<std::vector<std::string>>();
+    }
+    if (data.contains("health_endpoint")) {
+        spawn.health_endpoint = data["health_endpoint"].get<std::string>();
+    }
+
+    return spawn;
+}
+
+static json serialize_spawn_config(const SpawnConfig & spawn) {
+    json obj;
+    obj["command"] = spawn.command;
+    obj["proxy_endpoints"] = spawn.proxy_endpoints;
+    obj["health_endpoint"] = spawn.health_endpoint;
+    return obj;
+}
+
+const SpawnConfig & get_default_spawn() {
+    static const SpawnConfig spawn = [] {
+        SpawnConfig default_spawn = {
+            /*command          =*/ {"llama-server", "--ctx-size", "4096", "--n-gpu-layers", "99"},
+            /*proxy_endpoints =*/ {"/v1/", "/health", "/slots", "/props"},
+            /*health_endpoint =*/ "/health",
         };
 
         std::error_code ec;
         const std::string detected_path = detect_llama_server_binary();
         if (!detected_path.empty() && std::filesystem::exists(detected_path, ec) && !ec) {
             LOG_INF("Detected llama-server at %s\n", detected_path.c_str());
-            default_spawn[0] = detected_path;
+            default_spawn.command[0] = detected_path;
         } else {
             LOG_INF("Falling back to llama-server resolved via PATH\n");
         }
@@ -135,7 +160,7 @@ static void ensure_parent_directory(const std::string & path) {
 void write_config_file(const RouterConfig & cfg, const std::string & path) {
     json out;
     out["version"]       = cfg.version;
-    out["default_spawn"] = cfg.default_spawn;
+    out["default_spawn"] = serialize_spawn_config(cfg.default_spawn);
     out["router"] = {{"host", cfg.router.host},
                      {"port", cfg.router.port},
                      {"base_port", cfg.router.base_port},
@@ -155,8 +180,8 @@ void write_config_file(const RouterConfig & cfg, const std::string & path) {
         if (!m.group.empty()) {
             obj["group"] = m.group;
         }
-        if (!m.spawn.empty()) {
-            obj["spawn"] = m.spawn;
+        if (!is_spawn_empty(m.spawn)) {
+            obj["spawn"] = serialize_spawn_config(m.spawn);
         }
         out["models"].push_back(std::move(obj));
     }
@@ -204,7 +229,7 @@ RouterConfig load_config(const std::string & path) {
         cfg.version = data["version"].get<std::string>();
     }
     if (data.contains("default_spawn")) {
-        cfg.default_spawn = data["default_spawn"].get<std::vector<std::string>>();
+        cfg.default_spawn = parse_spawn_config(data["default_spawn"]);
     }
     if (data.contains("router")) {
         auto r = data["router"];
@@ -223,7 +248,7 @@ RouterConfig load_config(const std::string & path) {
             mc.state = m.value("state", "manual");
             mc.group = m.value("group", "");
             if (m.contains("spawn")) {
-                mc.spawn = m["spawn"].get<std::vector<std::string>>();
+                mc.spawn = parse_spawn_config(m["spawn"]);
             }
             cfg.models.push_back(std::move(mc));
         }
@@ -248,11 +273,14 @@ RouterConfig load_config(const std::string & path) {
             throw std::runtime_error("model path does not exist: " + path_to_check);
         }
 
-        if (!model.spawn.empty()) {
-            const std::string & cmd = model.spawn.front();
-            if (!cmd.empty() && cmd.find('/') != std::string::npos && !std::filesystem::exists(cmd, ec)) {
-                throw std::runtime_error("spawn command not executable: " + cmd);
-            }
+        const SpawnConfig & spawn = is_spawn_empty(model.spawn) ? cfg.default_spawn : model.spawn;
+        if (spawn.command.empty()) {
+            throw std::runtime_error("spawn command missing for model: " + model.name);
+        }
+
+        const std::string & cmd = spawn.command.front();
+        if (!cmd.empty() && cmd.find('/') != std::string::npos && !std::filesystem::exists(cmd, ec)) {
+            throw std::runtime_error("spawn command not executable: " + cmd);
         }
     }
 

tools/router/router-config.h

@@ -3,12 +3,22 @@
 #include <string>
 #include <vector>
 
+struct SpawnConfig {
+    std::vector<std::string> command;
+    std::vector<std::string> proxy_endpoints;
+    std::string              health_endpoint;
+};
+
+inline bool is_spawn_empty(const SpawnConfig & spawn) {
+    return spawn.command.empty() && spawn.proxy_endpoints.empty() && spawn.health_endpoint.empty();
+}
+
 struct ModelConfig {
     std::string              name;
     std::string              path;
     std::string              state;
     std::string              group;
-    std::vector<std::string> spawn;
+    SpawnConfig              spawn;
 };
 
 struct RouterOptions {
@@ -22,14 +32,14 @@ struct RouterOptions {
 
 struct RouterConfig {
     std::string              version;
-    std::vector<std::string> default_spawn;
+    SpawnConfig              default_spawn;
     RouterOptions            router;
     std::vector<ModelConfig> models;
 };
 
 std::string get_default_config_path();
 std::string expand_user_path(const std::string & path);
-const std::vector<std::string> & get_default_spawn();
+const SpawnConfig & get_default_spawn();
 const RouterOptions &             get_default_router_options();
 
 RouterConfig load_config(const std::string & path);

tools/router/router-endpoints.cpp

@@ -74,7 +74,8 @@ void register_routes(httplib::Server & server, RouterApp & app) {
             return;
         }
         LOG_INF("Proxying %s to last spawned model %s\n", req.path.c_str(), model.c_str());
-        proxy_request(req, res, app.upstream_for(model), app.get_config().router);
+        const auto spawn_cfg = app.get_spawn_config(model);
+        proxy_request(req, res, app.upstream_for(model), app.get_config().router, spawn_cfg.proxy_endpoints);
     };
 
     server.Get("/props", proxy_last_spawned);
@@ -93,7 +94,8 @@ void register_routes(httplib::Server & server, RouterApp & app) {
             return;
         }
         LOG_INF("Proxying %s for model %s\n", req.path.c_str(), model_name.c_str());
-        proxy_request(req, res, app.upstream_for(model_name), app.get_config().router);
+        const auto spawn_cfg = app.get_spawn_config(model_name);
+        proxy_request(req, res, app.upstream_for(model_name), app.get_config().router, spawn_cfg.proxy_endpoints);
     });
 
     server.Post("/v1/chat/completions", [&app](const httplib::Request & req, httplib::Response & res) {
@@ -114,7 +116,8 @@ void register_routes(httplib::Server & server, RouterApp & app) {
         }
 
         LOG_INF("Proxying chat completion for model %s\n", model.c_str());
-        proxy_request(req, res, app.upstream_for(model), app.get_config().router);
+        const auto spawn_cfg = app.get_spawn_config(model);
+        proxy_request(req, res, app.upstream_for(model), app.get_config().router, spawn_cfg.proxy_endpoints);
     });
 
     server.Post("/admin/reload", [&app](const httplib::Request & req, httplib::Response & res) {

tools/router/router-process.cpp

@@ -320,16 +320,21 @@ ProcessHandle spawn_process(const std::vector<std::string> & args) {
     return handle;
 }
 
-bool wait_for_backend_ready(int port, int timeout_ms, const ProcessHandle * process) {
+bool wait_for_backend_ready(int port, const std::string & health_endpoint, int timeout_ms, const ProcessHandle * process) {
     httplib::Client client("127.0.0.1:" + std::to_string(port));
     const auto      start = std::chrono::steady_clock::now();
     auto            next_log_ms = 0;
 
-    LOG_INF("Waiting up to %d ms for backend readiness on port %d\n", timeout_ms, port);
+    const std::string endpoint = health_endpoint.empty() ? "/health" : health_endpoint;
+
+    LOG_INF("Waiting up to %d ms for backend readiness on port %d (endpoint %s)\n",
+            timeout_ms,
+            port,
+            endpoint.c_str());
 
     while (true) {
         try {
-            auto res = client.Get("/health");
+            auto res = client.Get(endpoint.c_str());
             if (res && res->status == 200) {
                 LOG_INF("Backend on port %d reports ready\n", port);
                 return true;

tools/router/router-process.h

@@ -42,4 +42,7 @@ void         close_process(ProcessHandle & handle);
 void         terminate_process(ProcessHandle & handle);
 bool         wait_for_process_exit(const ProcessHandle & handle, int timeout_ms);
 ProcessHandle spawn_process(const std::vector<std::string> & args);
-bool         wait_for_backend_ready(int port, int timeout_ms, const ProcessHandle * process = nullptr);
+bool         wait_for_backend_ready(int port,
+                                    const std::string & health_endpoint,
+                                    int timeout_ms,
+                                    const ProcessHandle * process = nullptr);

tools/router/router-proxy.cpp

@@ -19,12 +19,27 @@ void copy_response_headers(const httplib::Headers & from, httplib::Response & to
         to.set_header(h.first, h.second);
     }
 }
+
+bool matches_any_endpoint(const std::string & path, const std::vector<std::string> & patterns) {
+    if (patterns.empty()) {
+        return true;
+    }
+
+    for (const auto & pattern : patterns) {
+        if (path.compare(0, pattern.size(), pattern) == 0) {
+            return true;
+        }
+    }
+
+    return false;
+}
 } // namespace
 
 bool proxy_request(const httplib::Request & req,
                    httplib::Response &       res,
                    const std::string &       upstream_base,
-                   const RouterOptions &     opts) {
+                   const RouterOptions &     opts,
+                   const std::vector<std::string> & proxy_endpoints) {
     if (upstream_base.empty()) {
         res.status = 502;
         res.set_content("{\"error\":\"missing upstream\"}", "application/json");
@@ -41,6 +56,13 @@ bool proxy_request(const httplib::Request & req,
 
     const std::string path = !req.target.empty() ? req.target : req.path;
 
+    if (!matches_any_endpoint(path, proxy_endpoints)) {
+        LOG_WRN("Request %s not proxied because it does not match configured endpoints\n", path.c_str());
+        res.status = 404;
+        res.set_content("{\"error\":\"endpoint not proxied\"}", "application/json");
+        return false;
+    }
+
     std::string content_type = req.get_header_value("Content-Type", "application/json");
 
     const auto accept_header = req.get_header_value("Accept");

tools/router/router-proxy.h

@@ -9,4 +9,5 @@
 bool proxy_request(const httplib::Request & req,
                    httplib::Response &       res,
                    const std::string &       upstream_base,
-                   const RouterOptions &     opts);
+                   const RouterOptions &     opts,
+                   const std::vector<std::string> & proxy_endpoints);

tools/router/router-scanner.cpp

@@ -142,8 +142,8 @@ std::vector<ModelConfig> scan_default_models() {
         mc.state = "auto";
         if (auto it_mmproj = mmproj_map.find(full_path); it_mmproj != mmproj_map.end()) {
             mc.spawn = get_default_spawn();
-            mc.spawn.push_back("--mmproj");
-            mc.spawn.push_back(it_mmproj->second);
+            mc.spawn.command.push_back("--mmproj");
+            mc.spawn.command.push_back(it_mmproj->second);
         }
 
         models.push_back(std::move(mc));