We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 5a914ff commit f626b71Copy full SHA for f626b71
convert-hf-to-gguf.py
@@ -2726,8 +2726,6 @@ def set_vocab(self):
2726
vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab()))
2727
assert max(tokenizer.get_vocab().values()) < vocab_size
2728
2729
- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.get_vocab().items()}
2730
-
2731
for token_id in range(vocab_size):
2732
piece = tokenizer._convert_id_to_token(token_id)
2733
if token_id == 0:
0 commit comments