GGUF model loading (#3571)

Co-authored-by: Rafal Sapala <[email protected]>
Co-authored-by: Miłosz Żeglarski <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: 4 people

external/lfs.patch

@@ -290,7 +290,7 @@ new file mode 100644
 index 000000000..cd3aa39fd
 --- /dev/null
 +++ b/src/libgit2/lfs_filter.c
-@@ -0,0 +1,497 @@
+@@ -0,0 +1,500 @@
 +/*
 +/ Copyright 2025 Intel Corporation
 +/
@@ -643,7 +643,7 @@ index 000000000..cd3aa39fd
 +		/* Remove a header curl would otherwise add by itself */
 +		chunk = curl_slist_append(chunk, "Accept: application/vnd.git-lfs+json");
 +		/* Add a custom header */
-+		chunk = curl_slist_append(chunk, "Content-type: application/json");
++		chunk = curl_slist_append(chunk, "Content-Type: application/vnd.git-lfs+json");
 +		/* set our custom set of headers */
 +		curl_easy_setopt(info_curl, CURLOPT_HTTPHEADER, chunk);
 +		/* First set the URL that is about to receive our POST. This URL
@@ -678,6 +678,7 @@ index 000000000..cd3aa39fd
 +		struct memory response = { 0 };
 +		curl_easy_setopt(
 +		        info_curl, CURLOPT_WRITEFUNCTION, write_callback);
++		curl_easy_setopt(info_curl, CURLOPT_USERAGENT,"git-lfs/3.5.0");
 +		curl_easy_setopt(
 +		        info_curl, CURLOPT_WRITEDATA, (void *)&response);
 +
@@ -711,7 +712,7 @@ index 000000000..cd3aa39fd
 +					   get_digit(la->lfs_size), 0,
 +			                   time(NULL), time(NULL) };
 +
-+		const char *href_regexp = "{\"href\":\"(.*)\"}}}]}";
++		const char *href_regexp = "\"download\"\\s*:\\s*\\{\\s*\"href\":\"([^\"]+)\"";
 +		if (get_lfs_info_match(&res_str, href_regexp) < 0) {
 +			/* always cleanup */
 +			curl_easy_cleanup(dl_curl);
@@ -724,6 +725,7 @@ index 000000000..cd3aa39fd
 +		curl_easy_setopt(dl_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA);
 +		curl_easy_setopt(dl_curl, CURLOPT_FOLLOWLOCATION, 1L);
 +		curl_easy_setopt(dl_curl, CURLOPT_USE_SSL, CURLUSESSL_ALL);
++		curl_easy_setopt(dl_curl, CURLOPT_USERAGENT,"git-lfs/3.5.0");
 +		curl_easy_setopt(dl_curl, CURLOPT_WRITEFUNCTION, file_write_callback);
 +		curl_easy_setopt(dl_curl, CURLOPT_WRITEDATA, (void *)&ftpfile);
 +
@@ -762,6 +764,7 @@ index 000000000..cd3aa39fd
 +
 +on_error:
 +	git__free(payload);
++	fprintf(stderr, "LFS download failed for file %s\n", la->full_path);
 +	return;
 +}
 +

src/BUILD

@@ -294,6 +294,7 @@ ovms_cc_library(
         "ovms_exit_codes",
     ],
     visibility = ["//visibility:public",],
+    additional_copts = COPTS_DROGON,
 )
 ovms_cc_library(
     name = "libovms_cliparser",
@@ -2660,10 +2661,12 @@ cc_test(
             ":inferencerequest_test",
             ":libtest_environment",
             ":libtest_gpuenvironment",
+            ":test_gguf_environment",
             ":network_utils_test",
             "//src:custom_nodes_common_buffersqueue",
             "@com_google_googletest//:gtest",
             ":pull_hf_model_test",
+            ":pull_gguf_hf_model_test",
             ":listdirectorymodels_test",
             ":graph_export_test",
             ":config_export_test",
@@ -2751,6 +2754,20 @@ cc_library(
     local_defines = COMMON_LOCAL_DEFINES,
     copts = COPTS_TESTS,
 )
+cc_library(
+    name = "test_gguf_environment",
+    linkstatic = 1,
+    alwayslink = True,
+    hdrs = ["test/gguf_environment.hpp",],
+    srcs = ["test/gguf_environment.cpp",],
+    linkopts = [],
+    deps = [
+            "libovmslogging",
+            "@com_google_googletest//:gtest",
+    ],
+    local_defines = COMMON_LOCAL_DEFINES,
+    copts = COPTS_TESTS,
+)
 cc_library(
     name = "inferencerequest_test",
     linkstatic = 1,
@@ -2788,6 +2805,25 @@ cc_library(
     local_defines = COMMON_LOCAL_DEFINES,
     copts = COPTS_TESTS,
 )
+cc_library(
+    name = "pull_gguf_hf_model_test",
+    linkstatic = 1,
+    alwayslink = True,
+    srcs = ["test/pull_gguf_hf_model_test.cpp"],
+    linkopts = [],
+    deps = [
+        "@libgit2_engine//:libgit2_engine",
+        "//src/servables_config_manager_module:listmodels",
+        ":test_utils",
+        ":test_gguf_environment",
+        "//src/pull_module:gguf_downloader",
+        "//src:ovms_lib",
+        "libovmsstring_utils",
+        "@com_google_googletest//:gtest",
+    ],
+    local_defines = COMMON_LOCAL_DEFINES,
+    copts = COPTS_TESTS,
+)
 cc_library(
     name = "listdirectorymodels_test",
     srcs = ["test/listdirectorymodels_test.cpp"],

src/capi_frontend/server_settings.hpp

@@ -34,6 +34,7 @@ enum GraphExportType : unsigned int {
 enum ModelDownlaodType : unsigned int {
     GIT_CLONE_DOWNLOAD,
     OPTIMUM_CLI_DOWNLOAD,
+    GGUF_DOWNLOAD,
     UNKNOWN_DOWNLOAD
 };
 
@@ -144,6 +145,7 @@ struct ImageGenerationGraphSettingsImpl {
 struct HFSettingsImpl {
     std::string targetDevice = "CPU";
     std::string sourceModel = "";
+    std::optional<std::string> ggufFilename;
     std::string downloadPath = "";
     bool overwriteModels = false;
     std::optional<std::string> extraQuantizationParams;

src/cli_parser.cpp

@@ -176,6 +176,10 @@ void CLIParser::parse(int argc, char** argv) {
                 "HF source model path",
                 cxxopts::value<std::string>(),
                 "HF_SOURCE")
+            ("gguf_filename",
+                "Name of the GGUF file",
+                cxxopts::value<std::string>(),
+                "GGUF_FILENAME")
             ("overwrite_models",
                 "Overwrite the model if it already exists in the models repository",
                 cxxopts::value<bool>()->default_value("false"),
@@ -459,7 +463,6 @@ void CLIParser::prepareModel(ModelsSettingsImpl& modelsSettings, HFSettingsImpl&
         modelsSettings.modelPath = result->operator[]("model_path").as<std::string>();
         modelsSettings.userSetSingleModelArguments.push_back("model_name");
     }
-
     if (result->count("max_sequence_number")) {
         modelsSettings.maxSequenceNumber = result->operator[]("max_sequence_number").as<uint32_t>();
         modelsSettings.userSetSingleModelArguments.push_back("max_sequence_number");
@@ -537,13 +540,17 @@ void CLIParser::prepareGraph(ServerSettingsImpl& serverSettings, HFSettingsImpl&
         } else {
             serverSettings.serverMode = HF_PULL_AND_START_MODE;
         }
-
+        if (result->count("gguf_filename")) {
+            hfSettings.ggufFilename = result->operator[]("gguf_filename").as<std::string>();
+            hfSettings.downloadType = GGUF_DOWNLOAD;
+        }
         if (result->count("overwrite_models"))
             hfSettings.overwriteModels = result->operator[]("overwrite_models").as<bool>();
         if (result->count("source_model")) {
             hfSettings.sourceModel = result->operator[]("source_model").as<std::string>();
-            // FIXME: Currently we use git clone only for OpenVINO, we will change this method of detection to parsing model files
-            if (!startsWith(toLower(serverSettings.hfSettings.sourceModel), toLower("OpenVINO/"))) {
+            // TODO: Currently we use git clone only for OpenVINO, we will change this method of detection to parsing model files
+            if (!startsWith(toLower(serverSettings.hfSettings.sourceModel), toLower("OpenVINO/")) &&
+                (hfSettings.ggufFilename == std::nullopt)) {
                 hfSettings.downloadType = OPTIMUM_CLI_DOWNLOAD;
             }
         }

src/config.cpp

@@ -128,8 +128,8 @@ bool Config::validate() {
             std::cerr << "Error: --task parameter not set." << std::endl;
             return false;
         }
-        if (serverSettings.hfSettings.downloadType != OPTIMUM_CLI_DOWNLOAD && !startsWith(toLower(serverSettings.hfSettings.sourceModel), toLower("OpenVINO/"))) {
-            std::cerr << "For now only OpenVINO models are supported in pulling mode";
+        if (serverSettings.hfSettings.downloadType == GIT_CLONE_DOWNLOAD && !startsWith(toLower(serverSettings.hfSettings.sourceModel), toLower("OpenVINO/"))) {
+            std::cerr << "For now only OpenVINO models are supported in pulling mode with git clone. Please use optimum download or gguf models instead." << std::endl;
             return false;
         }
         if (this->serverSettings.hfSettings.task == TEXT_GENERATION_GRAPH) {

src/graph_export/graph_export.cpp

@@ -54,9 +54,30 @@ namespace ovms {
 
 static const std::string OVMS_VERSION_GRAPH_LINE = std::string("# File created with: ") + PROJECT_NAME + std::string(" ") + PROJECT_VERSION + std::string("\n");
 
-static Status createTextGenerationGraphTemplate(const std::string& directoryPath, const TextGenGraphSettingsImpl& graphSettings) {
+static std::string constructModelsPath(const std::string& modelPath, const std::optional<std::string>& ggufFilenameOpt) {
+    std::string modelsPath;
+    if (ggufFilenameOpt.has_value()) {
+        modelsPath = FileSystem::joinPath({modelPath, ggufFilenameOpt.value()});
+#if _WIN32
+        // On Windows, file paths use backslashes ('\') as separators. However, the graph parser used in this project expects Unix-style paths with forward slashes ('/').
+        // If Windows-style backslashes are present, the parser may fail to locate files or misinterpret the path. To ensure compatibility, we replace all backslashes with forward slashes.
+        // This is safe because Windows APIs accept forward slashes in file paths.
+        if (FileSystem::getOsSeparator() != "/") {
+            std::replace(modelsPath.begin(), modelsPath.end(), '\\', '/');
+        }
+#endif
+    } else {
+        modelsPath = modelPath;
+    }
+    SPDLOG_TRACE("Models path: {}, modelPath:{}, ggufFilenameOpt:{}", modelsPath, modelPath, ggufFilenameOpt.value_or("std::nullopt"));
+    return modelsPath;
+}
+
+static Status createTextGenerationGraphTemplate(const std::string& directoryPath, const TextGenGraphSettingsImpl& graphSettings, const std::optional<std::string> ggufFilename) {
     std::ostringstream oss;
     oss << OVMS_VERSION_GRAPH_LINE;
+    std::string modelsPath = constructModelsPath(graphSettings.modelPath, ggufFilename);
+    SPDLOG_TRACE("modelsPath: {}, directoryPath: {}, ggufFilename: {}", modelsPath, directoryPath, ggufFilename.value_or("std::nullopt"));
     // clang-format off
     oss << R"(
     input_stream: "HTTP_REQUEST_PAYLOAD:input"
@@ -80,7 +101,7 @@ static Status createTextGenerationGraphTemplate(const std::string& directoryPath
             device: ")"
         << graphSettings.targetDevice << R"(",
             models_path: ")"
-        << graphSettings.modelPath << R"(",
+        << modelsPath << R"(",
             plugin_config: ')"
         << GraphExport::createPluginString(graphSettings.pluginConfig) << R"(',
             enable_prefix_caching: )"
@@ -134,6 +155,7 @@ static Status createTextGenerationGraphTemplate(const std::string& directoryPath
 #if (MEDIAPIPE_DISABLE == 0)
     ::mediapipe::CalculatorGraphConfig config;
     bool success = ::google::protobuf::TextFormat::ParseFromString(oss.str(), &config);
+    SPDLOG_TRACE("Generated pbtxt: {}", oss.str());
     if (!success) {
         SPDLOG_ERROR("Created graph config file couldn't be parsed - check used task parameters values.");
         return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID;
@@ -325,20 +347,20 @@ Status GraphExport::createServableConfig(const std::string& directoryPath, const
     auto status = LocalFileSystem::exists(directoryPath, &exists);
     if (!status.ok())
         return status;
-
-    bool is_dir = false;
-    status = LocalFileSystem::isDir(directoryPath, &is_dir);
-    if (!status.ok())
-        return status;
-
-    if (!is_dir) {
-        SPDLOG_ERROR("Graph path is not a directory: {}", directoryPath);
-        return StatusCode::PATH_INVALID;
+    if (!hfSettings.ggufFilename.has_value()) {
+        bool is_dir = false;
+        status = LocalFileSystem::isDir(directoryPath, &is_dir);
+        if (!status.ok())
+            return status;
+
+        if (!is_dir) {
+            SPDLOG_ERROR("Graph path is not a directory: {}", directoryPath);
+            return StatusCode::PATH_INVALID;
+        }
     }
-
     if (hfSettings.task == TEXT_GENERATION_GRAPH) {
         if (std::holds_alternative<TextGenGraphSettingsImpl>(hfSettings.graphSettings)) {
-            return createTextGenerationGraphTemplate(directoryPath, std::get<TextGenGraphSettingsImpl>(hfSettings.graphSettings));
+            return createTextGenerationGraphTemplate(directoryPath, std::get<TextGenGraphSettingsImpl>(hfSettings.graphSettings), hfSettings.ggufFilename);
         } else {
             SPDLOG_ERROR("Graph options not initialized for text generation.");
             return StatusCode::INTERNAL_ERROR;

src/llm/BUILD

@@ -177,6 +177,7 @@ ovms_cc_library(
         "//src:libovmsstatus",
         "//src:libovmsmediapipe_utils",
         "//src:libovmsprofiler",
+        "//src:libovmsstring_utils",
         "//src:libovmsfilesystem",
         "llmcalculator_cc_proto",
         ":openai_completions_api_handler",

src/llm/io_processing/base_generation_config_builder.cpp

@@ -15,6 +15,7 @@
 //*****************************************************************************
 
 #include <limits>
+#include <string>
 #include <openvino/genai/generation_config.hpp>
 #include "base_generation_config_builder.hpp"
 
@@ -29,6 +30,10 @@ void BaseGenerationConfigBuilder::setStructuralTagsConfig(const ov::genai::Struc
     }
 }
 
+void BaseGenerationConfigBuilder::addStopString(const std::string& decodedStopString) {
+    config.stop_strings.insert(decodedStopString);
+}
+
 void BaseGenerationConfigBuilder::validateStructuredOutputConfig(ov::genai::Tokenizer& tokenizer) {
     if (config.structured_output_config.has_value()) {
         config.structured_output_config.value().validate(tokenizer);

src/llm/io_processing/base_generation_config_builder.hpp

@@ -14,6 +14,8 @@
 // limitations under the License.
 //*****************************************************************************
 #pragma once
+#include <string>
+
 #include <openvino/genai/generation_config.hpp>
 #include <openvino/genai/tokenizer.hpp>
 #include "../apis/openai_request.hpp"
@@ -40,6 +42,11 @@ class BaseGenerationConfigBuilder {
 
     ov::genai::GenerationConfig& getConfig() { return config; }
 
+    /*
+    * Add stop string to generation config. Used when model server needs to add additional stop string that has not been provided in the request.
+    */
+    void addStopString(const std::string& decodedStopString);
+
     /*
     * Validates the structured output configuration, if exists.
     * Throws exception if validation fails.

src/llm/io_processing/generation_config_builder.hpp

@@ -70,5 +70,9 @@ class GenerationConfigBuilder {
     void parseConfigFromRequest(const OpenAIChatCompletionsRequest& request) {
         builder_impl->parseConfigFromRequest(request);
     }
+
+    void addStopString(const std::string& decodedStopString) {
+        builder_impl->addStopString(decodedStopString);
+    }
 };
 }  // namespace ovms