Add Deepseek-R1-Distill pre-tokenizer

Iwan Kawrakow · Iwan Kawrakow · commit c2624b2fd324 · 2025-01-23T13:10:03.000+02:00
diff --git a/src/llama.cpp b/src/llama.cpp
@@ -5591,7 +5591,7 @@ static void llm_load_vocab(
                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;
                 vocab.tokenizer_clean_spaces = false;
             } else if (
-                tokenizer_pre == "qwen2") {
+                tokenizer_pre == "qwen2" || tokenizer_pre == "deepseek-r1-qwen") {
                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_QWEN2;
                 vocab.tokenizer_clean_spaces = false;
             } else if (
