From b66cdce7d1ba34813f0cc72f2dd2acc9e072291b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Fri, 8 Aug 2025 14:49:51 +0200 Subject: [PATCH] enable -td and -tbd for server --- common/arg.cpp | 4 ++-- tools/server/server.cpp | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 0f01bb31454a4..14088cc6a7873 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -3134,7 +3134,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.speculative.cpuparams.n_threads = std::thread::hardware_concurrency(); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"-tbd", "--threads-batch-draft"}, "N", "number of threads to use during batch and prompt processing (default: same as --threads-draft)", @@ -3144,7 +3144,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.speculative.cpuparams_batch.n_threads = std::thread::hardware_concurrency(); } } - ).set_examples({LLAMA_EXAMPLE_SPECULATIVE})); + ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"-Cd", "--cpu-mask-draft"}, "M", "Draft model CPU affinity mask. Complements cpu-range-draft (default: same as --cpu-mask)", diff --git a/tools/server/server.cpp b/tools/server/server.cpp index a255d481a4d1c..0d6ba57b26da1 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -2011,6 +2011,9 @@ struct server_context { params_dft.cache_type_k = params_base.speculative.cache_type_k; params_dft.cache_type_v = params_base.speculative.cache_type_v; + params_dft.cpuparams.n_threads = params_base.speculative.cpuparams.n_threads; + params_dft.cpuparams_batch.n_threads = params_base.speculative.cpuparams_batch.n_threads; + llama_init_dft = common_init_from_params(params_dft); model_dft = llama_init_dft.model.get();