perf(core): revert mutex to optimize single-thread encoding speed

Author: wangzhaode

src/tokenizer.cpp

@@ -16,7 +16,6 @@
 #include <oniguruma.h>
 #include <utf8proc/utf8proc.h>
 #include <iostream>
-#include <mutex>
 #include "ujson.hpp"
 #include "jinja.hpp"
 
@@ -518,7 +517,6 @@ class BPEModel : public Model {
     std::unordered_map<std::string, int> vocab_;
     std::unordered_map<int, std::string> id_to_token_;
     std::unordered_map<std::pair<int, int>, int, PairHash> merges_;
-    mutable std::mutex cache_mutex_;
     mutable std::unordered_map<std::string, std::vector<int>> cache_;
 
     BPEModel(const std::map<std::string, int>& vocab,
@@ -543,11 +541,9 @@ class BPEModel : public Model {
 
     std::vector<int> tokenize(const std::string& text) const override {
         if (text.empty()) return {};
-        {
-            std::lock_guard<std::mutex> lock(cache_mutex_);
-            auto cit = cache_.find(text);
-            if (cit != cache_.end()) return cit->second;
-        }
+        auto cit = cache_.find(text);
+        if (cit != cache_.end()) return cit->second;
+
         std::vector<int> out;
         if (use_byte_level_) {
             static auto byte_map = create_bytes_char_map();
@@ -589,10 +585,7 @@ class BPEModel : public Model {
             int nid = token_to_id(m); if (nid == -1) break;
             out[best] = nid; out.erase(out.begin() + best + 1);
         }
-        {
-            std::lock_guard<std::mutex> lock(cache_mutex_);
-            cache_[text] = out;
-        }
+        cache_[text] = out;
         return out;
     }