Skip to content

Commit 301ba77

Browse files
authored
type fix
1 parent 551a64f commit 301ba77

File tree

1 file changed

+3
-3
lines changed

1 file changed

+3
-3
lines changed

convert_hf_to_gguf.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2653,15 +2653,15 @@ def set_vocab(self):
26532653
scores: list[float] = [-10000.0] * vocab_size
26542654
toktypes: list[int] = [gguf.TokenType.UNUSED] * vocab_size
26552655

2656-
def decode_grok_token(token: dict, toktype: gguf.TokenType) -> tuple[gguf.TokenType, int, str]:
2656+
def decode_grok_token(token: dict, toktype: gguf.TokenType) -> tuple[gguf.TokenType, int, bytes]:
26572657
tokid = token["token"]
26582658
tokb = token["bytes"]
26592659
try:
26602660
tokc = bytes(tokb).decode("utf-8")
2661-
except:
2661+
except Exception:
26622662
tokc = None
26632663
if len(tokb) == 1 or not tokc:
2664-
return gguf.TokenType.BYTE, tokid, "<0x{:02X}>".format(tokb[0])
2664+
return gguf.TokenType.BYTE, tokid, "<0x{:02X}>".format(tokb[0]).encode("utf-8")
26652665
else:
26662666
return toktype, tokid, tokc
26672667

0 commit comments

Comments
 (0)