glm 4.5 set eos/eog/eot token to <|user|>

Author: sammcj

convert_hf_to_gguf.py

@@ -6603,39 +6603,19 @@ def set_vocab(self):
         self.gguf_writer.add_token_types(toktypes)
 
         # Special tokens
-        # BOS should be [gMASK] (151331), EOS should be <|endoftext|> (151329) as per tokenizer analysis
-        special_vocab._set_special_token(
-            "eos", tokenizer.get_added_vocab()["<|endoftext|>"]  # 151329 - correct EOS token
-        )
-        special_vocab._set_special_token(
-            "eot", tokenizer.get_added_vocab()["<|endoftext|>"]  # 151329 - same as EOS
-        )
-        special_vocab._set_special_token(
-            "unk", tokenizer.get_added_vocab()["<|endoftext|>"]
-        )
-        special_vocab._set_special_token(
-            "bos", tokenizer.get_added_vocab()["[gMASK]"]  # 151331
-        )
+        # Note: Using <|endoftext|> (151329) for eos and eot causes endless generation
+        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"])  # 151331
+        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|user|>"])  # 151336 - end of
+        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])  # 151336 - same as EOS
+        special_vocab._set_special_token("eog", tokenizer.get_added_vocab()["<|user|>"])  # 151336 - same as EOS
+        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # 151329
         special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"])  # 151338
 
         if "<sop>" in tokenizer.get_added_vocab():
             special_vocab._set_special_token("sop", tokenizer.get_added_vocab()["<sop>"])  # 151333
         if "<eop>" in tokenizer.get_added_vocab():
             special_vocab._set_special_token("eop", tokenizer.get_added_vocab()["<eop>"])  # 151334
 
-        # TODO: clean up once decided on an approach to think and /nothink
-        #
-        # Previously:
-        # if "/nothink" in tokenizer.get_added_vocab():
-        #     special_vocab._set_special_token("nothink", tokenizer.get_added_vocab()["/nothink"])  # 151360
-        # Note: <think> and </think> are regular tokens (special=false in official config), not special tokens
-        #
-        # Latest thinking is:
-        # NOTE: /nothink token exists but causes generation issues as mentioned in
-        # https://huggingface.co/zai-org/GLM-4.5/discussions/9
-        # "it is a very special token. Even as input, it will be encoded into a special token, causing generation issues."
-        # Therefore we do NOT add it to avoid generation problems
-
         special_vocab.add_to_gguf(self.gguf_writer)
 
     def set_gguf_parameters(self):