Skip to content

Commit 634cf89

Browse files
CISCMinh141120
authored andcommitted
gguf-py : fix TemplateProcessing pair when bos/eos is missing (ggml-org#14312)
1 parent 2d597b3 commit 634cf89

File tree

1 file changed

+3
-3
lines changed

1 file changed

+3
-3
lines changed

gguf-py/gguf/vocab.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,9 +202,9 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool:
202202
if special_last != special_eos:
203203
logger.warning(f'Unknown trailing special token {special_last!r} in TemplateProcessing<single>')
204204
if tmpl_pair:
205-
seq_start = 1 if tmpl_pair[0].get('SpecialToken', {}).get('id') == special_first else 0
206-
seq_stop = -1 if tmpl_pair[-1].get('SpecialToken', {}).get('id') == special_last else None
207-
if seq_start == 0 or seq_stop is None:
205+
seq_start = 1 if special_first and tmpl_pair[0].get('SpecialToken', {}).get('id') == special_first else 0
206+
seq_stop = -1 if special_last and tmpl_pair[-1].get('SpecialToken', {}).get('id') == special_last else None
207+
if (special_first and seq_start == 0) or (special_last and seq_stop is None):
208208
logger.warning('TemplateProcessing<single> leading/trailing special tokens do not match TemplateProcessing<pair>')
209209
if tmpl_pair := tmpl_pair[slice(seq_start, seq_stop)]:
210210
tmpl_a = tmpl_pair[0].get('Sequence', {}).get('id')

0 commit comments

Comments
 (0)