feat: Update llama.cpp

Author: abetlen

llama_cpp/llama_cpp.py

@@ -284,6 +284,27 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa
 """BERT tokenizer based on WordPiece"""
 
 
+# // pre-tokenization types
+# enum llama_vocab_pre_type {
+#     LLAMA_VOCAB_PRE_TYPE_DEFAULT        = 0,
+#     LLAMA_VOCAB_PRE_TYPE_LLAMA3         = 1,
+#     LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM   = 2,
+#     LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,
+#     LLAMA_VOCAB_PRE_TYPE_FALCON         = 4,
+#     LLAMA_VOCAB_PRE_TYPE_MPT            = 5,
+#     LLAMA_VOCAB_PRE_TYPE_STARCODER      = 6,
+#     LLAMA_VOCAB_PRE_TYPE_GPT2           = 7,
+# };
+LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0
+LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1
+LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2
+LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3
+LLAMA_VOCAB_PRE_TYPE_FALCON = 4
+LLAMA_VOCAB_PRE_TYPE_MPT = 5
+LLAMA_VOCAB_PRE_TYPE_STARCODER = 6
+LLAMA_VOCAB_PRE_TYPE_GPT2 = 7
+
+
 # // note: these values should be synchronized with ggml_rope
 # // TODO: maybe move this enum to ggml.h (ggml_rope_type)
 # enum llama_rope_type {

tests/test_llama.py

@@ -6,7 +6,7 @@
 
 import llama_cpp
 
-MODEL = "./vendor/llama.cpp/models/ggml-vocab-llama.gguf"
+MODEL = "./vendor/llama.cpp/models/ggml-vocab-llama-spm.gguf"
 
 
 def test_llama_cpp_tokenization():