From c044bf33f7e835ca6a723ccc97848de25dba5164 Mon Sep 17 00:00:00 2001 From: Soumyadip Sarkar Date: Wed, 6 Aug 2025 04:05:10 +0530 Subject: [PATCH 1/2] fix: `meta_sep` token in `encoding.rs` --- src/encoding.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/encoding.rs b/src/encoding.rs index c58e8b8..030c96f 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -68,7 +68,7 @@ impl FormattingToken { FormattingToken::Channel => "<|channel|>", FormattingToken::BeginUntrusted => "<|untrusted|>", FormattingToken::EndUntrusted => "<|end_untrusted|>", - FormattingToken::MetaSep => "<|channel|>", + FormattingToken::MetaSep => "<|meta_sep|>", FormattingToken::MetaEnd => "<|meta_end|>", } } From c505a03e9c9a388a511b6125756097eee523742a Mon Sep 17 00:00:00 2001 From: Soumyadip Sarkar Date: Wed, 6 Aug 2025 04:50:17 +0530 Subject: [PATCH 2/2] fix: `meta_sep` token and add to registry --- src/registry.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/registry.rs b/src/registry.rs index 6d8a98f..4fe585d 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -66,6 +66,8 @@ pub fn load_harmony_encoding(name: HarmonyEncodingName) -> anyhow::Result"), (FormattingToken::BeginUntrusted, "<|untrusted|>"), (FormattingToken::EndUntrusted, "<|end_untrusted|>"), + (FormattingToken::MetaSep, "<|meta_sep|>"), + (FormattingToken::MetaEnd, "<|meta_end|>"), ]), stop_formatting_tokens: HashSet::from([ FormattingToken::EndMessageDoneSampling, @@ -107,6 +109,8 @@ pub async fn load_harmony_encoding(name: HarmonyEncodingName) -> anyhow::Result< (FormattingToken::EndMessageAssistantToTool, "<|call|>"), (FormattingToken::BeginUntrusted, "<|untrusted|>"), (FormattingToken::EndUntrusted, "<|end_untrusted|>"), + (FormattingToken::MetaSep, "<|meta_sep|>"), + (FormattingToken::MetaEnd, "<|meta_end|>"), ]), stop_formatting_tokens: HashSet::from([ FormattingToken::EndMessageDoneSampling,