Skip to content

Commit 6cb886b

Browse files
committed
Revert "model : gpt-oss revert removal of user defined tokens"
This reverts commit 6d75412.
1 parent 6d75412 commit 6cb886b

File tree

1 file changed

+0
-7
lines changed

1 file changed

+0
-7
lines changed

src/llama-vocab.cpp

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2339,13 +2339,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
23392339
}
23402340
}
23412341

2342-
// @ngxson : quick hack for gpt-oss, always render these tokens
2343-
for (const auto & t : token_to_id) {
2344-
if (t.first == "<|channel|>" || t.first == "<|message|>" || t.first == "<|start|>") {
2345-
id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_USER_DEFINED;
2346-
}
2347-
}
2348-
23492342
// sanity checks
23502343
if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {
23512344
special_eog_ids.insert(special_eos_id);

0 commit comments

Comments
 (0)