We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent a854897 commit d7f340bCopy full SHA for d7f340b
gguf-py/gguf/vocab.py
@@ -160,6 +160,8 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool:
160
special_cls = (tokenizer_config or {}).get('cls_token')
161
special_eos = (tokenizer_config or {}).get('eos_token')
162
special_sep = (tokenizer_config or {}).get('sep_token')
163
+ if not special_bos and special_cls and tokenizer_config:
164
+ tokenizer_config['bos_token'] = special_bos = special_cls
165
if not special_eos and special_sep and tokenizer_config:
166
tokenizer_config['eos_token'] = special_eos = special_sep
167
post_processor = tokenizer.get('post_processor', {})
0 commit comments