From db2d53e9ca3d03dece6551d76ff960bad16b7e03 Mon Sep 17 00:00:00 2001 From: Kai Aoki Date: Thu, 16 Oct 2025 16:44:51 +0900 Subject: [PATCH 1/8] Supports multi-byte character image file paths --- tools/mtmd/mtmd-helper.cpp | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index 686f42f3960fe..fac910ffdf9a8 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -437,13 +437,38 @@ mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigne } mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname) { - std::vector buf; +#ifdef _WIN32 + // Convert UTF-8 to UTF-16 for Windows + int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0); + if (!wlen) { + fprintf(stderr, "Unable to convert filename to UTF-16: %s\n", fname); + return nullptr; + } + + std::vector wfname(wlen); + wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wlen); + if (!wlen) { + fprintf(stderr, "Unable to convert filename to UTF-16: %s\n", fname); + return nullptr; + } + + // Open file with UTF-16 filename + FILE * f = _wfopen(wfname.data(), L"rb"); + if (!f) { + fprintf(stderr, "Unable to open file %s: %s\n", fname, strerror(errno)); + return nullptr; + } +#else + // On non-Windows platforms, use fopen directly FILE * f = fopen(fname, "rb"); if (!f) { - LOG_ERR("Unable to open file %s: %s\n", fname, strerror(errno)); + fprintf(stderr, "Unable to open file %s: %s\n", fname, strerror(errno)); return nullptr; } +#endif + // Read file content + std::vector buf; fseek(f, 0, SEEK_END); long file_size = ftell(f); fseek(f, 0, SEEK_SET); @@ -451,8 +476,9 @@ mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * size_t n_read = fread(buf.data(), 1, file_size, f); fclose(f); + if (n_read != (size_t)file_size) { - LOG_ERR("Failed to read entire file %s", fname); + fprintf(stderr, "Failed to read entire file %s\n", fname); return nullptr; } From 751e06073e1d4100e478ee5cf748e9b48d138d8a Mon Sep 17 00:00:00 2001 From: Kai Aoki Date: Thu, 16 Oct 2025 16:45:52 +0900 Subject: [PATCH 2/8] Perform the same console init process as llama-cli --- tools/mtmd/mtmd-cli.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 5fde6ca0c32ae..e1522a4015446 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -270,6 +270,9 @@ int main(int argc, char ** argv) { int n_predict = params.n_predict < 0 ? INT_MAX : params.n_predict; + console::init(params.simple_io, params.use_color); + atexit([]() { console::cleanup(); }); + // Ctrl+C handling { #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) From 57fc675ea4f38b01271a71b4ac3e6e426b750040 Mon Sep 17 00:00:00 2001 From: Kai Aoki Date: Thu, 16 Oct 2025 17:11:02 +0900 Subject: [PATCH 3/8] Supports multi-byte character mmproj file paths --- tools/mtmd/clip.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 98e68af27a690..41d3615f4c016 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -28,6 +28,13 @@ #include #include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + struct clip_logger_state g_logger_state = {GGML_LOG_LEVEL_CONT, clip_log_callback_default, NULL}; enum ffn_op_type { @@ -2762,7 +2769,21 @@ struct clip_model_loader { { std::vector read_buf; +#ifdef _WIN32 + // Convert UTF-8 to UTF-16 for Windows + int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0); + if (!wlen) { + throw std::runtime_error(string_format("%s: failed to convert filename to UTF-16: %s\n", __func__, fname.c_str())); + } + std::vector wfname(wlen); + wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen); + if (!wlen) { + throw std::runtime_error(string_format("%s: failed to convert filename to UTF-16: %s\n", __func__, fname.c_str())); + } + auto fin = std::ifstream(wfname.data(), std::ios::binary); +#else auto fin = std::ifstream(fname, std::ios::binary); +#endif if (!fin) { throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str())); } From 0bb73a37f809962d6f1603f19e6c8b3e40614535 Mon Sep 17 00:00:00 2001 From: Kai Aoki Date: Thu, 16 Oct 2025 17:49:35 +0900 Subject: [PATCH 4/8] Added handling for non-ASCII arguments on Windows --- common/arg.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/common/arg.cpp b/common/arg.cpp index c0b718071127d..ded8ca68ad5a7 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -14,6 +14,7 @@ # define NOMINMAX #endif #include +#include #endif #define JSON_ASSERT GGML_ASSERT @@ -1626,7 +1627,44 @@ static void add_rpc_devices(const std::string & servers) { } } +#ifdef _WIN32 +struct Utf8Argv { + std::vector buf; + std::vector ptrs; +}; + +static Utf8Argv make_utf8_argv_from_winapi() { + Utf8Argv out; + int wargc = 0; + LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc); + if (!wargv) return out; + + out.buf.reserve(wargc); + for (int i = 0; i < wargc; ++i) { + int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr); + if (n <= 0) { out.buf.emplace_back(); continue; } + auto& s = out.buf.emplace_back(); + s.resize(static_cast(n - 1)); + (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr); + } + LocalFree(wargv); + + out.ptrs.reserve(out.buf.size() + 1); + for (auto& s : out.buf) out.ptrs.push_back(s.data()); + out.ptrs.push_back(nullptr); + return out; +} +#endif + bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) { +#ifdef _WIN32 + auto utf8 = make_utf8_argv_from_winapi(); + if (!utf8.ptrs.empty()) { + argc = static_cast(utf8.buf.size()); + argv = utf8.ptrs.data(); + } +#endif + auto ctx_arg = common_params_parser_init(params, ex, print_usage); const common_params params_org = ctx_arg.params; // the example can modify the default params From ba38a3114a0c94fb8ea7fead30563abc06722499 Mon Sep 17 00:00:00 2001 From: Kai Aoki Date: Thu, 16 Oct 2025 19:04:39 +0900 Subject: [PATCH 5/8] Revert "Supports multi-byte character mmproj file paths" This reverts commit 57fc675ea4f38b01271a71b4ac3e6e426b750040. --- tools/mtmd/clip.cpp | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 41d3615f4c016..98e68af27a690 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -28,13 +28,6 @@ #include #include -#ifdef _WIN32 -#ifndef NOMINMAX -#define NOMINMAX -#endif -#include -#endif - struct clip_logger_state g_logger_state = {GGML_LOG_LEVEL_CONT, clip_log_callback_default, NULL}; enum ffn_op_type { @@ -2769,21 +2762,7 @@ struct clip_model_loader { { std::vector read_buf; -#ifdef _WIN32 - // Convert UTF-8 to UTF-16 for Windows - int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0); - if (!wlen) { - throw std::runtime_error(string_format("%s: failed to convert filename to UTF-16: %s\n", __func__, fname.c_str())); - } - std::vector wfname(wlen); - wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen); - if (!wlen) { - throw std::runtime_error(string_format("%s: failed to convert filename to UTF-16: %s\n", __func__, fname.c_str())); - } - auto fin = std::ifstream(wfname.data(), std::ios::binary); -#else auto fin = std::ifstream(fname, std::ios::binary); -#endif if (!fin) { throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str())); } From dddf4ce02eb91871b663c4b17c8fd661ded2c37f Mon Sep 17 00:00:00 2001 From: Kai Aoki Date: Thu, 16 Oct 2025 19:06:19 +0900 Subject: [PATCH 6/8] Reapply "Supports multi-byte character mmproj file paths" This reverts commit ba38a3114a0c94fb8ea7fead30563abc06722499. --- tools/mtmd/clip.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 98e68af27a690..41d3615f4c016 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -28,6 +28,13 @@ #include #include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + struct clip_logger_state g_logger_state = {GGML_LOG_LEVEL_CONT, clip_log_callback_default, NULL}; enum ffn_op_type { @@ -2762,7 +2769,21 @@ struct clip_model_loader { { std::vector read_buf; +#ifdef _WIN32 + // Convert UTF-8 to UTF-16 for Windows + int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0); + if (!wlen) { + throw std::runtime_error(string_format("%s: failed to convert filename to UTF-16: %s\n", __func__, fname.c_str())); + } + std::vector wfname(wlen); + wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen); + if (!wlen) { + throw std::runtime_error(string_format("%s: failed to convert filename to UTF-16: %s\n", __func__, fname.c_str())); + } + auto fin = std::ifstream(wfname.data(), std::ios::binary); +#else auto fin = std::ifstream(fname, std::ios::binary); +#endif if (!fin) { throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str())); } From 4aa58976a100b25ccb8b10d92ebc4df8f7fa72d4 Mon Sep 17 00:00:00 2001 From: Kai Aoki Date: Thu, 16 Oct 2025 19:06:28 +0900 Subject: [PATCH 7/8] Revert "Added handling for non-ASCII arguments on Windows" This reverts commit 0bb73a37f809962d6f1603f19e6c8b3e40614535. --- common/arg.cpp | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index ded8ca68ad5a7..c0b718071127d 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -14,7 +14,6 @@ # define NOMINMAX #endif #include -#include #endif #define JSON_ASSERT GGML_ASSERT @@ -1627,44 +1626,7 @@ static void add_rpc_devices(const std::string & servers) { } } -#ifdef _WIN32 -struct Utf8Argv { - std::vector buf; - std::vector ptrs; -}; - -static Utf8Argv make_utf8_argv_from_winapi() { - Utf8Argv out; - int wargc = 0; - LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc); - if (!wargv) return out; - - out.buf.reserve(wargc); - for (int i = 0; i < wargc; ++i) { - int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr); - if (n <= 0) { out.buf.emplace_back(); continue; } - auto& s = out.buf.emplace_back(); - s.resize(static_cast(n - 1)); - (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr); - } - LocalFree(wargv); - - out.ptrs.reserve(out.buf.size() + 1); - for (auto& s : out.buf) out.ptrs.push_back(s.data()); - out.ptrs.push_back(nullptr); - return out; -} -#endif - bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) { -#ifdef _WIN32 - auto utf8 = make_utf8_argv_from_winapi(); - if (!utf8.ptrs.empty()) { - argc = static_cast(utf8.buf.size()); - argv = utf8.ptrs.data(); - } -#endif - auto ctx_arg = common_params_parser_init(params, ex, print_usage); const common_params params_org = ctx_arg.params; // the example can modify the default params From 91eb1606021a954488db7e5f952dfba35ead280c Mon Sep 17 00:00:00 2001 From: Kai Aoki Date: Fri, 17 Oct 2025 23:37:18 +0900 Subject: [PATCH 8/8] use LOG_ERR --- tools/mtmd/mtmd-helper.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index fac910ffdf9a8..e55a4843be10a 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -441,28 +441,28 @@ mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * // Convert UTF-8 to UTF-16 for Windows int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0); if (!wlen) { - fprintf(stderr, "Unable to convert filename to UTF-16: %s\n", fname); + LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname); return nullptr; } std::vector wfname(wlen); wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wlen); if (!wlen) { - fprintf(stderr, "Unable to convert filename to UTF-16: %s\n", fname); + LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname); return nullptr; } // Open file with UTF-16 filename FILE * f = _wfopen(wfname.data(), L"rb"); if (!f) { - fprintf(stderr, "Unable to open file %s: %s\n", fname, strerror(errno)); + LOG_ERR("Unable to open file %s: %s\n", fname, strerror(errno)); return nullptr; } #else // On non-Windows platforms, use fopen directly FILE * f = fopen(fname, "rb"); if (!f) { - fprintf(stderr, "Unable to open file %s: %s\n", fname, strerror(errno)); + LOG_ERR("Unable to open file %s: %s\n", fname, strerror(errno)); return nullptr; } #endif @@ -478,7 +478,7 @@ mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fclose(f); if (n_read != (size_t)file_size) { - fprintf(stderr, "Failed to read entire file %s\n", fname); + LOG_ERR("Failed to read entire file %s\n", fname); return nullptr; }