Skip to content

Commit 6fda4fe

Browse files
fix flake errors
1 parent ecb345f commit 6fda4fe

File tree

1 file changed

+3
-4
lines changed

1 file changed

+3
-4
lines changed

convert_hf_to_gguf.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5582,15 +5582,14 @@ def set_vocab(self):
55825582
merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)
55835583
if len(merged) == 2:
55845584
merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
5585-
55865585

55875586
# Build token list
55885587
vocab_size = self.hparams["vocab_size"]
55895588
special_tokens = tokenizer.special_tokens
55905589
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
55915590
tokens: list[str] = []
55925591
toktypes: list[int] = []
5593-
5592+
55945593
for i in range(vocab_size):
55955594
if i not in reverse_vocab:
55965595
tokens.append(f"[PAD{i}]")
@@ -5602,13 +5601,13 @@ def set_vocab(self):
56025601
toktypes.append(gguf.TokenType.CONTROL)
56035602
else:
56045603
toktypes.append(gguf.TokenType.NORMAL)
5605-
5604+
56065605
self.gguf_writer.add_tokenizer_model("gpt2")
56075606
self.gguf_writer.add_tokenizer_pre(tokpre)
56085607
self.gguf_writer.add_token_list(tokens)
56095608
self.gguf_writer.add_token_types(toktypes)
56105609
self.gguf_writer.add_token_merges(merges)
5611-
5610+
56125611
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
56135612
special_vocab.add_to_gguf(self.gguf_writer)
56145613
else:

0 commit comments

Comments
 (0)