We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 42d8b4c commit 69aa327Copy full SHA for 69aa327
convert_hf_to_gguf.py
@@ -5597,8 +5597,11 @@ def set_vocab(self):
5597
5598
# 3. Generate the tokens and toktypes lists
5599
vocab_size = self.hparams["vocab_size"]
5600
+ prit("====================================")
5601
print(f'self.hparams["vocab_size"] = {vocab_size}')
5602
print(f'vocab_size = {vocab_size}')
5603
5604
+
5605
assert tokenizer.vocab_size == vocab_size
5606
special_tokens = tokenizer.special_tokens
5607
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
0 commit comments