File tree Expand file tree Collapse file tree 1 file changed +2
-2
lines changed
Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Original file line number Diff line number Diff line change @@ -2833,14 +2833,14 @@ def set_vocab(self):
28332833 self .gguf_writer .add_token_types (toktypes )
28342834
28352835 special_vocab = gguf .SpecialVocab (self .dir_model , n_vocab = len (tokens ))
2836-
2836+
28372837 tokenizer_config_file = self .dir_model / 'tokenizer_config.json'
28382838 if tokenizer_config_file .is_file ():
28392839 with open (tokenizer_config_file , "r" , encoding = "utf-8" ) as f :
28402840 tokenizer_config_json = json .load (f )
28412841 if "add_prefix_space" in tokenizer_config_json :
28422842 self .gguf_writer .add_add_space_prefix (tokenizer_config_json ["add_prefix_space" ])
2843-
2843+
28442844 if "added_tokens_decoder" in tokenizer_config_json :
28452845 for token_id , token_data in tokenizer_config_json ["added_tokens_decoder" ].items ():
28462846 if token_data .get ("special" ):
You can’t perform that action at this time.
0 commit comments