From 6617f3cc4b74d8ca7a5674f8abc923e372702398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Tue, 17 Jun 2025 12:17:46 +0200 Subject: [PATCH 1/5] Mtmd: add a way to select device for vision encoder --- tools/mtmd/clip.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 9146c9e9c4481..fe08e41dfc37b 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -384,9 +384,16 @@ struct clip_ctx { if (!backend_cpu) { throw std::runtime_error("failed to initialize CPU backend"); } - backend = ctx_params.use_gpu - ? ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr) - : nullptr; + backend = nullptr; + auto backend_name = std::getenv("MTMD_BACKEND_DEVICE"); + if(backend_name != nullptr && ctx_params.use_gpu) { + backend = ggml_backend_init_by_name(backend_name, nullptr); + } + if(!backend){ + backend = ctx_params.use_gpu + ? ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr) + : nullptr; + } if (backend) { LOG_INF("%s: CLIP using %s backend\n", __func__, ggml_backend_name(backend)); From f9fb321e429738366e06c0f70f337a897218e6ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Tue, 17 Jun 2025 12:27:10 +0200 Subject: [PATCH 2/5] simplify --- tools/mtmd/clip.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index fe08e41dfc37b..e26816d5542e4 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -385,14 +385,14 @@ struct clip_ctx { throw std::runtime_error("failed to initialize CPU backend"); } backend = nullptr; - auto backend_name = std::getenv("MTMD_BACKEND_DEVICE"); - if(backend_name != nullptr && ctx_params.use_gpu) { - backend = ggml_backend_init_by_name(backend_name, nullptr); - } - if(!backend){ - backend = ctx_params.use_gpu - ? ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr) - : nullptr; + if(ctx_params.use_gpu) { + auto backend_name = std::getenv("MTMD_BACKEND_DEVICE"); + if(backend_name != nullptr) { + backend = ggml_backend_init_by_name(backend_name, nullptr); + } + if(!backend){ + backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); + } } if (backend) { From ffd0bd897375d7d919d62d0dd51c31b7b049ec11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Thu, 26 Jun 2025 12:02:27 +0200 Subject: [PATCH 3/5] format --- tools/mtmd/clip.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index e26816d5542e4..d04af9ac21e5c 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -385,12 +385,12 @@ struct clip_ctx { throw std::runtime_error("failed to initialize CPU backend"); } backend = nullptr; - if(ctx_params.use_gpu) { + if (ctx_params.use_gpu) { auto backend_name = std::getenv("MTMD_BACKEND_DEVICE"); - if(backend_name != nullptr) { + if (backend_name != nullptr) { backend = ggml_backend_init_by_name(backend_name, nullptr); } - if(!backend){ + if (!backend) { backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); } } From baf7e3e803f6de57eadc2b58734a1e92083eb31f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Tue, 22 Jul 2025 12:08:53 +0200 Subject: [PATCH 4/5] Warn user if manual device selection failed --- tools/mtmd/clip.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index d04af9ac21e5c..57fb53d356ce2 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -389,6 +389,9 @@ struct clip_ctx { auto backend_name = std::getenv("MTMD_BACKEND_DEVICE"); if (backend_name != nullptr) { backend = ggml_backend_init_by_name(backend_name, nullptr); + if (!backend) { + LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name); + } } if (!backend) { backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); From 01e448c0802b06738d298a2430b4589486011a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Tue, 22 Jul 2025 12:09:28 +0200 Subject: [PATCH 5/5] initialize backend to nullptr --- tools/mtmd/clip.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 57fb53d356ce2..be191404cfc75 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -367,8 +367,8 @@ struct clip_ctx { std::vector backend_ptrs; std::vector backend_buft; - ggml_backend_t backend; - ggml_backend_t backend_cpu; + ggml_backend_t backend = nullptr; + ggml_backend_t backend_cpu = nullptr; ggml_backend_buffer_ptr buf; int max_nodes = 8192; @@ -384,7 +384,6 @@ struct clip_ctx { if (!backend_cpu) { throw std::runtime_error("failed to initialize CPU backend"); } - backend = nullptr; if (ctx_params.use_gpu) { auto backend_name = std::getenv("MTMD_BACKEND_DEVICE"); if (backend_name != nullptr) {