add qwen tokenizer

Author: leejet

clip.hpp

@@ -6,7 +6,7 @@
 
 /*================================================== CLIPTokenizer ===================================================*/
 
-std::pair<std::unordered_map<std::string, float>, std::string> extract_and_remove_lora(std::string text) {
+__STATIC_INLINE__ std::pair<std::unordered_map<std::string, float>, std::string> extract_and_remove_lora(std::string text) {
     std::regex re("<lora:([^:]+):([^>]+)>");
     std::smatch matches;
     std::unordered_map<std::string, float> filename2multiplier;
@@ -31,7 +31,7 @@ std::pair<std::unordered_map<std::string, float>, std::string> extract_and_remov
     return std::make_pair(filename2multiplier, text);
 }
 
-std::vector<std::pair<int, std::u32string>> bytes_to_unicode() {
+__STATIC_INLINE__ std::vector<std::pair<int, std::u32string>> bytes_to_unicode() {
     std::vector<std::pair<int, std::u32string>> byte_unicode_pairs;
     std::set<int> byte_set;
     for (int b = static_cast<int>('!'); b <= static_cast<int>('~'); ++b) {
@@ -398,6 +398,7 @@ class CLIPTokenizer {
             }
             for (auto& token : matches) {
                 std::string token_str = token.str();
+                LOG_DEBUG("%s", token_str.c_str());
                 std::u32string utf32_token;
                 for (int i = 0; i < token_str.length(); i++) {
                     unsigned char b = token_str[i];

examples/cli/main.cpp

@@ -27,6 +27,8 @@
 
 #include "avi_writer.h"
 
+#include "qwen.hpp"
+
 #if defined(_WIN32)
 #define NOMINMAX
 #include <windows.h>
@@ -1138,6 +1140,20 @@ bool load_images_from_dir(const std::string dir,
 
 int main(int argc, const char* argv[]) {
     SDParams params;
+    params.verbose = true;
+    sd_set_log_callback(sd_log_cb, (void*)&params);
+    auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
+        return false;
+    };
+    // auto tokenizer = CLIPTokenizer();
+    auto tokenizer = Qwen::Qwen2Tokenizer();
+    std::string text("a lovely cat");
+    auto tokens = tokenizer.encode(text, on_new_token_cb);
+    for (auto token : tokens) {
+        std::cout << token << " ";
+    }
+    std::cout << std::endl;
+    exit(1);
     parse_args(argc, argv, params);
     params.sample_params.guidance.slg.layers                 = params.skip_layers.data();
     params.sample_params.guidance.slg.layer_count            = params.skip_layers.size();

model.cpp

@@ -16,6 +16,7 @@
 #include "stable-diffusion.h"
 #include "util.h"
 #include "vocab.hpp"
+#include "vocab_qwen.hpp"
 #include "vocab_umt5.hpp"
 
 #include "ggml-alloc.h"
@@ -1939,6 +1940,11 @@ std::string ModelLoader::load_merges() {
     return merges_utf8_str;
 }
 
+std::string ModelLoader::load_qwen2_merges() {
+    std::string merges_utf8_str(reinterpret_cast<const char*>(qwen2_merges_utf8_c_str), sizeof(qwen2_merges_utf8_c_str));
+    return merges_utf8_str;
+}
+
 std::string ModelLoader::load_t5_tokenizer_json() {
     std::string json_str(reinterpret_cast<const char*>(t5_tokenizer_json_str), sizeof(t5_tokenizer_json_str));
     return json_str;

model.h

@@ -258,6 +258,7 @@ class ModelLoader {
     ~ModelLoader() = default;
 
     static std::string load_merges();
+    static std::string load_qwen2_merges();
     static std::string load_t5_tokenizer_json();
     static std::string load_umt5_tokenizer_json();
 };