File tree Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Original file line number Diff line number Diff line change @@ -199,10 +199,10 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool:
199199 special_eos = special_last
200200 elif special_last != special_eos :
201201 if 'eot' not in self .special_token_types :
202- self .special_token_types = self .special_token_types + ('eot' , )
202+ self .special_token_types = tuple ( self .special_token_types ) + ('eot' , )
203203 tokenizer_config ['eot_token' ] = special_eos
204204 elif 'eom' not in self .special_token_types :
205- self .special_token_types = self .special_token_types + ('eom' , )
205+ self .special_token_types = tuple ( self .special_token_types ) + ('eom' , )
206206 tokenizer_config ['eom_token' ] = special_eos
207207 else :
208208 logger .warning (f'Overriding special token { special_eos !r} with { special_last !r} without EOT/EOM fallback!' )
You can’t perform that action at this time.
0 commit comments