Skip to content

Commit 6d75412

Browse files
committed
model : gpt-oss revert removal of user defined tokens
1 parent f058384 commit 6d75412

File tree

1 file changed

+7
-0
lines changed

1 file changed

+7
-0
lines changed

src/llama-vocab.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2339,6 +2339,13 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
23392339
}
23402340
}
23412341

2342+
// @ngxson : quick hack for gpt-oss, always render these tokens
2343+
for (const auto & t : token_to_id) {
2344+
if (t.first == "<|channel|>" || t.first == "<|message|>" || t.first == "<|start|>") {
2345+
id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_USER_DEFINED;
2346+
}
2347+
}
2348+
23422349
// sanity checks
23432350
if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {
23442351
special_eog_ids.insert(special_eos_id);

0 commit comments

Comments
 (0)