|
| 1 | +#include "chat-peg-parser.h" |
| 2 | + |
| 3 | +#include <nlohmann/json.hpp> |
| 4 | + |
| 5 | +using json = nlohmann::json; |
| 6 | + |
| 7 | +static std::string_view trim_trailing_space(std::string_view sv) { |
| 8 | + while (!sv.empty() && std::isspace(static_cast<unsigned char>(sv.back()))) { |
| 9 | + sv.remove_suffix(1); |
| 10 | + } |
| 11 | + return sv; |
| 12 | +} |
| 13 | + |
| 14 | +void common_chat_peg_mapper::from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result) { |
| 15 | + arena.visit(result, [this](const common_peg_ast_node & node) { |
| 16 | + map(node); |
| 17 | + }); |
| 18 | +} |
| 19 | + |
| 20 | +void common_chat_peg_mapper::map(const common_peg_ast_node & node) { |
| 21 | + bool is_reasoning = node.tag == common_chat_peg_builder::REASONING; |
| 22 | + bool is_content = node.tag == common_chat_peg_builder::CONTENT; |
| 23 | + |
| 24 | + if (is_reasoning) { |
| 25 | + result.reasoning_content = std::string(trim_trailing_space(node.text)); |
| 26 | + } |
| 27 | + |
| 28 | + if (is_content) { |
| 29 | + result.content = std::string(trim_trailing_space(node.text)); |
| 30 | + } |
| 31 | +} |
| 32 | + |
| 33 | +void common_chat_peg_native_mapper::map(const common_peg_ast_node & node) { |
| 34 | + common_chat_peg_mapper::map(node); |
| 35 | + |
| 36 | + bool is_tool_open = node.tag == common_chat_peg_native_builder::TOOL_OPEN; |
| 37 | + bool is_tool_name = node.tag == common_chat_peg_native_builder::TOOL_NAME; |
| 38 | + bool is_tool_id = node.tag == common_chat_peg_native_builder::TOOL_ID; |
| 39 | + bool is_tool_args = node.tag == common_chat_peg_native_builder::TOOL_ARGS; |
| 40 | + |
| 41 | + if (is_tool_open) { |
| 42 | + result.tool_calls.emplace_back(); |
| 43 | + current_tool = &result.tool_calls.back(); |
| 44 | + } |
| 45 | + |
| 46 | + if (is_tool_id && current_tool) { |
| 47 | + current_tool->id = std::string(trim_trailing_space(node.text)); |
| 48 | + } |
| 49 | + |
| 50 | + if (is_tool_name && current_tool) { |
| 51 | + current_tool->name = std::string(trim_trailing_space(node.text)); |
| 52 | + } |
| 53 | + |
| 54 | + if (is_tool_args && current_tool) { |
| 55 | + current_tool->arguments = std::string(trim_trailing_space(node.text)); |
| 56 | + } |
| 57 | +} |
| 58 | + |
| 59 | +void common_chat_peg_constructed_mapper::map(const common_peg_ast_node & node) { |
| 60 | + common_chat_peg_mapper::map(node); |
| 61 | + |
| 62 | + bool is_tool_open = node.tag == common_chat_peg_constructed_builder::TOOL_OPEN; |
| 63 | + bool is_tool_name = node.tag == common_chat_peg_constructed_builder::TOOL_NAME; |
| 64 | + bool is_tool_close = node.tag == common_chat_peg_constructed_builder::TOOL_CLOSE; |
| 65 | + bool is_arg_open = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_OPEN; |
| 66 | + bool is_arg_close = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_CLOSE; |
| 67 | + bool is_arg_name = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_NAME; |
| 68 | + bool is_arg_string = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_STRING_VALUE; |
| 69 | + bool is_arg_json = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_JSON_VALUE; |
| 70 | + |
| 71 | + if (is_tool_open) { |
| 72 | + result.tool_calls.emplace_back(); |
| 73 | + current_tool = &result.tool_calls.back(); |
| 74 | + arg_count = 0; |
| 75 | + } |
| 76 | + |
| 77 | + if (is_tool_name) { |
| 78 | + current_tool->name = std::string(node.text); |
| 79 | + current_tool->arguments = "{"; |
| 80 | + } |
| 81 | + |
| 82 | + if (is_arg_open) { |
| 83 | + needs_closing_quote = false; |
| 84 | + } |
| 85 | + |
| 86 | + if (is_arg_name && current_tool) { |
| 87 | + if (arg_count > 0) { |
| 88 | + current_tool->arguments += ","; |
| 89 | + } |
| 90 | + current_tool->arguments += json(trim_trailing_space(node.text)).dump() + ":"; |
| 91 | + ++arg_count; |
| 92 | + } |
| 93 | + |
| 94 | + if (is_arg_string && current_tool) { |
| 95 | + // Serialize to JSON, but exclude the end quote |
| 96 | + std::string dumped = json(node.text).dump(); |
| 97 | + current_tool->arguments += dumped.substr(0, dumped.size() - 1); |
| 98 | + needs_closing_quote = true; |
| 99 | + } |
| 100 | + |
| 101 | + if (is_arg_close && current_tool) { |
| 102 | + if (needs_closing_quote) { |
| 103 | + current_tool->arguments += "\""; |
| 104 | + } |
| 105 | + } |
| 106 | + |
| 107 | + if (is_arg_json && current_tool) { |
| 108 | + current_tool->arguments += std::string(trim_trailing_space(node.text)); |
| 109 | + } |
| 110 | + |
| 111 | + if (is_tool_close && current_tool) { |
| 112 | + current_tool->arguments += "}"; |
| 113 | + } |
| 114 | +} |
0 commit comments