We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 9ecf1bd commit 5592c08Copy full SHA for 5592c08
convert_hf_to_gguf.py
@@ -5049,6 +5049,7 @@ def set_vocab(self):
5049
special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
5050
# exclude glm-edge 1.5B & 4B
5051
if self.hparams.get("partial_rotary_factor", 1.0) == 0.5:
5052
+ # GLM4 model has no bos token set in its tokenizer config, set it manually to [gMASK]
5053
special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"])
5054
special_vocab.add_to_gguf(self.gguf_writer)
5055
0 commit comments