We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent f2d876a commit b17e981Copy full SHA for b17e981
convert_hf_to_gguf.py
@@ -3668,7 +3668,7 @@ def _xlmroberta_set_vocab(self) -> None:
3668
toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size
3669
3670
if isinstance(tokenizer, SentencePieceProcessor):
3671
- for token_id in range(vocab_size):
+ for token_id in range(tokenizer.vocab_size()):
3672
piece = tokenizer.IdToPiece(token_id)
3673
text = piece.encode("utf-8")
3674
score = tokenizer.GetScore(token_id)
0 commit comments