Fix Compiler Warnings C4333 and C5038 (#986)

* fix warnings

* pin safetensors version

---------

Co-authored-by: Sayan Shaw <[email protected]>

Author: sayanshaw24

operators/tokenizer/trietree.hpp

@@ -18,7 +18,7 @@ class TrieTree {
   static constexpr int kMaxTokenLength_ = 128;
   static constexpr ValueT kInvalidId_ = static_cast<ValueT>(invalid_id);
 
-  TrieTree(CharT ch = 0) : ch_(ch), value_(std::nullopt) {}
+  TrieTree(CharT ch = 0) : value_(std::nullopt), ch_(ch) {}
 
   void Add(const std::basic_string<CharT>& key, int idx = 0,
            const std::optional<ValueT>& value = std::nullopt) noexcept {

operators/tokenizer/ugm_kernels.hpp

@@ -796,21 +796,26 @@ class SpmUgmDecoder {
   // Helper: Encode a wchar_t as UTF-8
   std::string EncodeUTF8(wchar_t wc) const {
     std::string out;
-    if (wc < 0x80) {
-      out += static_cast<char>(wc);
-    } else if (wc < 0x800) {
-      out += static_cast<char>((wc >> 6) | 0xC0);
-      out += static_cast<char>((wc & 0x3F) | 0x80);
-    } else if (wc < 0x10000) {
-      out += static_cast<char>((wc >> 12) | 0xE0);
-      out += static_cast<char>(((wc >> 6) & 0x3F) | 0x80);
-      out += static_cast<char>((wc & 0x3F) | 0x80);
+
+    // Promote wchar_t to uint32_t to avoid data loss from shift operations and silence warning C4333
+    uint32_t u = static_cast<uint32_t>(wc);
+
+    if (u < 0x80) {
+      out += static_cast<char>(u);
+    } else if (u < 0x800) {
+      out += static_cast<char>((u >> 6) | 0xC0);
+      out += static_cast<char>((u & 0x3F) | 0x80);
+    } else if (u < 0x10000) {
+      out += static_cast<char>((u >> 12) | 0xE0);
+      out += static_cast<char>(((u >> 6) & 0x3F) | 0x80);
+      out += static_cast<char>((u & 0x3F) | 0x80);
     } else {
-      out += static_cast<char>((wc >> 18) | 0xF0);
-      out += static_cast<char>(((wc >> 12) & 0x3F) | 0x80);
-      out += static_cast<char>(((wc >> 6) & 0x3F) | 0x80);
-      out += static_cast<char>((wc & 0x3F) | 0x80);
+      out += static_cast<char>((u >> 18) | 0xF0);
+      out += static_cast<char>(((u >> 12) & 0x3F) | 0x80);
+      out += static_cast<char>(((u >> 6) & 0x3F) | 0x80);
+      out += static_cast<char>((u & 0x3F) | 0x80);
     }
+
     return out;
   }
 

requirements-dev.txt

@@ -5,6 +5,7 @@ onnx             >=1.9.0
 protobuf         < 4.0.0
 # multiple versions of onnxruntime are supported, but only one can be installed at a time
 onnxruntime      >=1.12.0
+safetensors      < 0.4.3
 transformers     < 4.50.0
 tensorflow_text  >=2.5.0;python_version < '3.11'
 requests         >=2.26.0