We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent cf87c76 commit f582b84Copy full SHA for f582b84
convert_hf_to_gguf.py
@@ -2649,7 +2649,7 @@ def set_vocab(self):
2649
tokenizer = json.load(f)
2650
2651
vocab_size = tokenizer["vocab_size"]
2652
- tokens: list[str] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]
+ tokens: list[str] = [f"[PAD{i}]" for i in range(vocab_size)]
2653
scores: list[float] = [-10000.0] * vocab_size
2654
toktypes: list[int] = [gguf.TokenType.UNUSED] * vocab_size
2655
0 commit comments