Update llama.cpp

Author: abetlen

llama_cpp/llama.py

@@ -455,7 +455,7 @@ def detokenize(self, tokens: List[int]) -> bytes:
             output += bytes(buffer[:n])
         # NOTE: Llama1 models automatically added a space at the start of the prompt
         # this line removes a leading space if the first token is a beginning of sentence token
-        return output[1:] if len(tokens) > 0 and tokens[0] == self.token_bos() else output
+        return output
 
     def set_cache(self, cache: Optional[BaseLlamaCache]):
         """Set the cache.

tests/test_llama.py

@@ -14,16 +14,16 @@ def test_llama_cpp_tokenization():
 
     tokens = llama.tokenize(text)
     assert tokens[0] == llama.token_bos()
-    assert tokens == [1, 15043, 2787]
+    assert tokens == [1, 10994, 2787]
     detokenized = llama.detokenize(tokens)
     assert detokenized == text
 
     tokens = llama.tokenize(text, add_bos=False)
     assert tokens[0] != llama.token_bos()
-    assert tokens == [15043, 2787]
+    assert tokens == [10994, 2787]
 
     detokenized = llama.detokenize(tokens)
-    assert detokenized != text
+    assert detokenized == text
 
 
 @pytest.mark.skip(reason="bug in tokenization where leading space is always inserted even if not after eos")