add parsing

Author: aldehir

common/parsers/harmony.cpp

@@ -1,13 +1,73 @@
 #include "harmony.h"
+#include "regex-partial.h"
 
 harmony_msg_parser::harmony_msg_parser(common_chat_msg_parser & builder)
-    : builder(builder)
-{
-    // TODO
-}
+    : builder(builder) {}
 
 void harmony_msg_parser::parse() {
     // TODO @ngxson : this won't work with --special enabled, we should fix that
-    builder.try_parse_reasoning("<|channel|>analysis<|message|>", "<|start|>assistant<|channel|>final<|message|>");
-    builder.add_content(builder.consume_rest());
+    //builder.try_parse_reasoning("<|channel|>analysis<|message|>", "<|start|>assistant<|channel|>final<|message|>");
+    //builder.add_content(builder.consume_rest());
+    channel();
+}
+
+void harmony_msg_parser::channel() {
+    if (builder.try_consume_literal("<|channel|>")) {
+        static const common_regex channel_type_regexp("(final|analysis|commentary)");
+        if (auto res = builder.try_consume_regex(channel_type_regexp)) {
+            auto type = builder.str(res->groups[0]);
+            if (type == "analysis") {
+                analysis();
+            } else if (type == "final") {
+                final();
+            } else if (type == "commentary") {
+                commentary();
+            }
+        }
+    }
+}
+
+void harmony_msg_parser::analysis() {
+    if (builder.try_consume_literal("<|message|>")) {
+        static const common_regex end("<\\|end\\|>");
+        if (auto res = builder.try_find_regex(end, std::string::npos, false)) {
+            builder.add_reasoning_content(res->prelude);
+            if (builder.try_consume_literal("<|start|>")) {
+                start();
+            }
+        } else {
+            builder.add_reasoning_content(builder.consume_rest());
+        }
+    }
+}
+
+void harmony_msg_parser::start() {
+    if (builder.try_consume_literal("assistant")) {
+        channel();
+    }
+}
+
+void harmony_msg_parser::final() {
+    if (builder.try_consume_literal("<|message|>")) {
+        builder.add_content(builder.consume_rest());
+    }
+}
+
+void harmony_msg_parser::commentary() {
+    if (builder.try_consume_literal(" to=")) {
+        user_function();
+    }
+}
+
+void harmony_msg_parser::user_function() {
+    static const common_regex tool_call_regex(
+        "functions\\.([a-zA-Z_][a-zA-Z0-9_]*)\\s*(<\\|constrain\\|>([a-z]+))?<\\|message\\|>"
+    );
+    if (auto res = builder.try_consume_regex(tool_call_regex)) {
+        auto name = builder.str(res->groups[1]);
+        //auto constrain_type = builder.str(res->groups[3]);
+        auto args = builder.consume_rest();
+
+        builder.add_tool_call(name, "", args);
+    }
 }

common/parsers/harmony.h

@@ -13,4 +13,12 @@ class harmony_msg_parser {
     harmony_msg_parser(common_chat_msg_parser & builder);
 
     void parse();
+
+  private:
+    void channel();
+    void analysis();
+    void final();
+    void commentary();
+    void start();
+    void user_function();
 };

src/llama-vocab.cpp

@@ -5,6 +5,7 @@
 #include "llama-impl.h"
 #include "llama-model-loader.h"
 
+#include "llama.h"
 #include "unicode.h"
 
 #include <algorithm>
@@ -2339,13 +2340,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             }
         }
 
-        // @aldehir : check if this is still needed
-        // // @ngxson : quick hack for gpt-oss, always render these tokens
-        // for (const auto & t : token_to_id) {
-        //     if (t.first == "<|channel|>" || t.first == "<|message|>" || t.first == "<|start|>") {
-        //         id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_USER_DEFINED;
-        //     }
-        // }
+        // @ngxson : quick hack for gpt-oss, always render these tokens
+        for (const auto & t : token_to_id) {
+            if (t.first == "<|channel|>" || t.first == "<|message|>" || t.first == "<|start|>") {
+                id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_USER_DEFINED;
+            }
+        }
 
         // sanity checks
         if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {