Skip to content

Commit 839a7d9

Browse files
oluwandabiraqnixsynapse
oluwandabira
authored and
qnixsynapse
committed
server: Add ability to mount server at prefix (ggml-org#14544)
* Add server_prefix * Correct server path env * Rename cli flag to --api-prefix * Change all to api_prefix
1 parent 61a57b8 commit 839a7d9

File tree

3 files changed

+39
-31
lines changed

3 files changed

+39
-31
lines changed

‎common/arg.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2734,6 +2734,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
27342734
params.public_path = value;
27352735
}
27362736
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_STATIC_PATH"));
2737+
add_opt(common_arg(
2738+
{"--api-prefix"}, "PREFIX",
2739+
string_format("prefix path the server serves from, without the trailing slash (default: %s)", params.api_prefix.c_str()),
2740+
[](common_params & params, const std::string & value) {
2741+
params.api_prefix = value;
2742+
}
2743+
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_API_PREFIX"));
27372744
add_opt(common_arg(
27382745
{"--no-webui"},
27392746
string_format("Disable the Web UI (default: %s)", params.webui ? "enabled" : "disabled"),

‎common/common.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,7 @@ struct common_params {
370370

371371
std::string hostname = "127.0.0.1";
372372
std::string public_path = ""; // NOLINT
373+
std::string api_prefix = ""; // NOLINT
373374
std::string chat_template = ""; // NOLINT
374375
bool use_jinja = false; // NOLINT
375376
bool enable_chat_template = true;

‎tools/server/server.cpp‎

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4806,14 +4806,14 @@ int main(int argc, char ** argv) {
48064806
// register static assets routes
48074807
if (!params.public_path.empty()) {
48084808
// Set the base directory for serving static files
4809-
bool is_found = svr->set_mount_point("/", params.public_path);
4809+
bool is_found = svr->set_mount_point(params.api_prefix + "/", params.public_path);
48104810
if (!is_found) {
48114811
LOG_ERR("%s: static assets path not found: %s\n", __func__, params.public_path.c_str());
48124812
return 1;
48134813
}
48144814
} else {
48154815
// using embedded static index.html
4816-
svr->Get("/", [](const httplib::Request & req, httplib::Response & res) {
4816+
svr->Get(params.api_prefix + "/", [](const httplib::Request & req, httplib::Response & res) {
48174817
if (req.get_header_value("Accept-Encoding").find("gzip") == std::string::npos) {
48184818
res.set_content("Error: gzip is not supported by this browser", "text/plain");
48194819
} else {
@@ -4829,37 +4829,37 @@ int main(int argc, char ** argv) {
48294829
}
48304830

48314831
// register API routes
4832-
svr->Get ("/health", handle_health); // public endpoint (no API key check)
4833-
svr->Get ("/metrics", handle_metrics);
4834-
svr->Get ("/props", handle_props);
4835-
svr->Post("/props", handle_props_change);
4836-
svr->Post("/api/show", handle_api_show);
4837-
svr->Get ("/models", handle_models); // public endpoint (no API key check)
4838-
svr->Get ("/v1/models", handle_models); // public endpoint (no API key check)
4839-
svr->Get ("/api/tags", handle_models); // ollama specific endpoint. public endpoint (no API key check)
4840-
svr->Post("/completion", handle_completions); // legacy
4841-
svr->Post("/completions", handle_completions);
4842-
svr->Post("/v1/completions", handle_completions_oai);
4843-
svr->Post("/chat/completions", handle_chat_completions);
4844-
svr->Post("/v1/chat/completions", handle_chat_completions);
4845-
svr->Post("/api/chat", handle_chat_completions); // ollama specific endpoint
4846-
svr->Post("/infill", handle_infill);
4847-
svr->Post("/embedding", handle_embeddings); // legacy
4848-
svr->Post("/embeddings", handle_embeddings);
4849-
svr->Post("/v1/embeddings", handle_embeddings_oai);
4850-
svr->Post("/rerank", handle_rerank);
4851-
svr->Post("/reranking", handle_rerank);
4852-
svr->Post("/v1/rerank", handle_rerank);
4853-
svr->Post("/v1/reranking", handle_rerank);
4854-
svr->Post("/tokenize", handle_tokenize);
4855-
svr->Post("/detokenize", handle_detokenize);
4856-
svr->Post("/apply-template", handle_apply_template);
4832+
svr->Get (params.api_prefix + "/health", handle_health); // public endpoint (no API key check)
4833+
svr->Get (params.api_prefix + "/metrics", handle_metrics);
4834+
svr->Get (params.api_prefix + "/props", handle_props);
4835+
svr->Post(params.api_prefix + "/props", handle_props_change);
4836+
svr->Post(params.api_prefix + "/api/show", handle_api_show);
4837+
svr->Get (params.api_prefix + "/models", handle_models); // public endpoint (no API key check)
4838+
svr->Get (params.api_prefix + "/v1/models", handle_models); // public endpoint (no API key check)
4839+
svr->Get (params.api_prefix + "/api/tags", handle_models); // ollama specific endpoint. public endpoint (no API key check)
4840+
svr->Post(params.api_prefix + "/completion", handle_completions); // legacy
4841+
svr->Post(params.api_prefix + "/completions", handle_completions);
4842+
svr->Post(params.api_prefix + "/v1/completions", handle_completions_oai);
4843+
svr->Post(params.api_prefix + "/chat/completions", handle_chat_completions);
4844+
svr->Post(params.api_prefix + "/v1/chat/completions", handle_chat_completions);
4845+
svr->Post(params.api_prefix + "/api/chat", handle_chat_completions); // ollama specific endpoint
4846+
svr->Post(params.api_prefix + "/infill", handle_infill);
4847+
svr->Post(params.api_prefix + "/embedding", handle_embeddings); // legacy
4848+
svr->Post(params.api_prefix + "/embeddings", handle_embeddings);
4849+
svr->Post(params.api_prefix + "/v1/embeddings", handle_embeddings_oai);
4850+
svr->Post(params.api_prefix + "/rerank", handle_rerank);
4851+
svr->Post(params.api_prefix + "/reranking", handle_rerank);
4852+
svr->Post(params.api_prefix + "/v1/rerank", handle_rerank);
4853+
svr->Post(params.api_prefix + "/v1/reranking", handle_rerank);
4854+
svr->Post(params.api_prefix + "/tokenize", handle_tokenize);
4855+
svr->Post(params.api_prefix + "/detokenize", handle_detokenize);
4856+
svr->Post(params.api_prefix + "/apply-template", handle_apply_template);
48574857
// LoRA adapters hotswap
4858-
svr->Get ("/lora-adapters", handle_lora_adapters_list);
4859-
svr->Post("/lora-adapters", handle_lora_adapters_apply);
4858+
svr->Get (params.api_prefix + "/lora-adapters", handle_lora_adapters_list);
4859+
svr->Post(params.api_prefix + "/lora-adapters", handle_lora_adapters_apply);
48604860
// Save & load slots
4861-
svr->Get ("/slots", handle_slots);
4862-
svr->Post("/slots/:id_slot", handle_slots_action);
4861+
svr->Get (params.api_prefix + "/slots", handle_slots);
4862+
svr->Post(params.api_prefix + "/slots/:id_slot", handle_slots_action);
48634863

48644864
//
48654865
// Start the server

0 commit comments

Comments
 (0)