Enable faster prompt processing with mainline llama.cpp GGUFs (#409)

ikawrakow · Iwan Kawrakow · web-flow · commit f27cd4054223 · 2025-05-12T07:49:51.000+03:00
* Enable MLA-3 in crippled GGUFs: WIP

* Enable MLA-3 in crippled GGUFs: seems to work

* Add newly created tensors to model.tensors_by_name

Else they don't get run-time repacked.

---------

Co-authored-by: Iwan Kawrakow &lt;iwan.kawrakow@gmail.com&gt;
diff --git a/common/common.cpp b/common/common.cpp
@@ -2334,6 +2334,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &
     if (params.n_gpu_layers != -1) {
         mparams.n_gpu_layers = params.n_gpu_layers;
     }
+    mparams.mla             = params.mla_attn;
     mparams.rpc_servers     = params.rpc_servers.c_str();
     mparams.main_gpu        = params.main_gpu;
     mparams.split_mode      = params.split_mode;
diff --git a/include/llama.h b/include/llama.h
@@ -325,6 +325,7 @@ extern "C" {
 
     struct llama_model_params {
         int32_t n_gpu_layers; // number of layers to store in VRAM
+        int32_t mla;          // MLA implementation to use (only applicable to DeepSeek models at this point)
         enum llama_split_mode split_mode; // how to split the model across multiple GPUs
 
         // main_gpu interpretation depends on split_mode:
diff --git a/src/llama.cpp b/src/llama.cpp

Original file line number	Diff line number	Diff line change
`@@ -2334,6 +2334,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &`
`2334`	`2334`	`if (params.n_gpu_layers != -1) {`
`2335`	`2335`	`mparams.n_gpu_layers = params.n_gpu_layers;`
`2336`	`2336`	`}`
	`2337`	`+ mparams.mla = params.mla_attn;`
`2337`	`2338`	`mparams.rpc_servers = params.rpc_servers.c_str();`
`2338`	`2339`	`mparams.main_gpu = params.main_gpu;`
`2339`	`2340`	`mparams.split_mode = params.split_mode;`