feat(llama.cpp): allow to set kv-overrides (#5745)

Signed-off-by: Ettore Di Giacinto <[email protected]>

Author: mudler

backend/backend.proto

@@ -258,6 +258,8 @@ message ModelOptions {
   repeated GrammarTrigger GrammarTriggers = 65;
 
   bool Reranking = 71;
+
+  repeated string Overrides = 72;
 }
 
 message Result {

backend/cpp/llama/grpc-server.cpp

@@ -12,6 +12,7 @@
 
 #include "backend.pb.h"
 #include "backend.grpc.pb.h"
+#include "common.h"
 #include <getopt.h>
 #include <grpcpp/ext/proto_server_reflection_plugin.h>
 #include <grpcpp/grpcpp.h>
@@ -260,6 +261,13 @@ static void params_parse(const backend::ModelOptions* request,
         }
     }
 
+    // Add kv_overrides
+    if (request->overrides_size() > 0) {
+        for (int i = 0; i < request->overrides_size(); i++) {
+            string_parse_kv_override(request->overrides(i).c_str(), params.kv_overrides);
+        }
+    }
+
     // TODO: Add yarn
 
     if (!request->tensorsplit().empty()) {

core/backend/options.go

@@ -143,6 +143,7 @@ func grpcModelOpts(c config.BackendConfig) *pb.ModelOptions {
 		CLIPModel:            c.Diffusers.ClipModel,
 		CLIPSubfolder:        c.Diffusers.ClipSubFolder,
 		Options:              c.Options,
+		Overrides:            c.Overrides,
 		CLIPSkip:             int32(c.Diffusers.ClipSkip),
 		ControlNet:           c.Diffusers.ControlNet,
 		ContextSize:          int32(ctxSize),

core/config/backend_config.go

@@ -70,7 +70,8 @@ type BackendConfig struct {
 	Description string `yaml:"description"`
 	Usage       string `yaml:"usage"`
 
-	Options []string `yaml:"options"`
+	Options   []string `yaml:"options"`
+	Overrides []string `yaml:"overrides"`
 }
 
 // Pipeline defines other models to use for audio-to-audio