fixed unicode paths

Author: LostRuins

model_adapter.cpp

@@ -15,6 +15,7 @@
 #include "gguf.h"
 
 #include <chrono>
+#include <filesystem>
 
 static auto bench_timer = std::chrono::high_resolution_clock().now();
 
@@ -86,7 +87,12 @@ void print_tok_vec(std::vector<float> &embd)
  {
     std::vector<char> f_buf(1024*1024);
 
-    auto fin = std::ifstream(fname, std::ios::binary);
+    #ifdef _WIN32
+        std::filesystem::path fpath = std::filesystem::u8path(fname);
+    #else
+        std::filesystem::path fpath = std::filesystem::path(fname);
+    #endif
+    auto fin = std::ifstream(fpath, std::ios::binary);
     fin.rdbuf()->pubsetbuf(f_buf.data(), f_buf.size());
     if (!fin) {
         fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());

otherarch/sdcpp/ggml_extend.hpp

@@ -19,6 +19,7 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
+#include <filesystem>
 
 #include "ggml-alloc.h"
 #include "ggml-backend.h"
@@ -218,7 +219,12 @@ __STATIC_INLINE__ void print_ggml_tensor(struct ggml_tensor* tensor, bool shape_
 }
 
 __STATIC_INLINE__ ggml_tensor* load_tensor_from_file(ggml_context* ctx, const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    #ifdef _WIN32
+        std::filesystem::path fpath = std::filesystem::u8path(file_path);
+    #else
+        std::filesystem::path fpath = std::filesystem::path(file_path);
+    #endif
+    std::ifstream file(fpath, std::ios::binary);
     if (!file.is_open()) {
         LOG_ERROR("failed to open '%s'", file_path.c_str());
         return NULL;

otherarch/sdcpp/model.cpp

@@ -5,6 +5,7 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
+#include <filesystem>
 
 #include "model.h"
 #include "stable-diffusion.h"
@@ -907,7 +908,12 @@ bool is_zip_file(const std::string& file_path) {
 }
 
 bool is_gguf_file(const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    #ifdef _WIN32
+        std::filesystem::path fpath = std::filesystem::u8path(file_path);
+    #else
+        std::filesystem::path fpath = std::filesystem::path(file_path);
+    #endif
+    std::ifstream file(fpath, std::ios::binary);
     if (!file.is_open()) {
         return false;
     }
@@ -928,7 +934,12 @@ bool is_gguf_file(const std::string& file_path) {
 }
 
 bool is_safetensors_file(const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    #ifdef _WIN32
+        std::filesystem::path fpath = std::filesystem::u8path(file_path);
+    #else
+        std::filesystem::path fpath = std::filesystem::path(file_path);
+    #endif
+    std::ifstream file(fpath, std::ios::binary);
     if (!file.is_open()) {
         return false;
     }
@@ -1052,7 +1063,12 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const
     LOG_DEBUG("init from '%s'", file_path.c_str());
     file_paths_.push_back(file_path);
     size_t file_index = file_paths_.size() - 1;
-    std::ifstream file(file_path, std::ios::binary);
+    #ifdef _WIN32
+        std::filesystem::path fpath = std::filesystem::u8path(file_path);
+    #else
+        std::filesystem::path fpath = std::filesystem::path(file_path);
+    #endif
+    std::ifstream file(fpath, std::ios::binary);
     if (!file.is_open()) {
         LOG_ERROR("failed to open '%s'", file_path.c_str());
         return false;
@@ -1809,7 +1825,12 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, ggml_backend
         std::string file_path = file_paths_[file_index];
         LOG_DEBUG("loading tensors from %s\n", file_path.c_str());
 
-        std::ifstream file(file_path, std::ios::binary);
+        #ifdef _WIN32
+            std::filesystem::path fpath = std::filesystem::u8path(file_path);
+        #else
+            std::filesystem::path fpath = std::filesystem::path(file_path);
+        #endif
+        std::ifstream file(fpath, std::ios::binary);
         if (!file.is_open()) {
             LOG_ERROR("failed to open '%s'", file_path.c_str());
             return false;

tools/mtmd/clip.cpp

@@ -44,6 +44,7 @@
 #include <array>
 #include <numeric>
 #include <functional>
+#include <filesystem>
 
 struct clip_logger_state g_logger_state = {GGML_LOG_LEVEL_CONT, clip_log_callback_default, NULL};
 
@@ -2107,7 +2108,12 @@ struct clip_model_loader {
         {
             std::vector<uint8_t> read_buf;
 
-            auto fin = std::ifstream(fname, std::ios::binary);
+            #ifdef _WIN32
+            std::filesystem::path fpath = std::filesystem::u8path(fname);
+            #else
+                std::filesystem::path fpath = std::filesystem::path(fname);
+            #endif
+            auto fin = std::ifstream(fpath, std::ios::binary);
             if (!fin) {
                 throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str()));
             }