bug-fix: don't attempt to log partial parsed messages to avoid crash due to unfinished UTF-8 sequences

Author: kallewoof

common/chat.cpp

@@ -1944,7 +1944,7 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co
         }
     }
     auto msg = builder.result();
-    if (msg.content == truncate_incomplete_utf8(msg.content)) {
+    if (!is_partial) {
         LOG_DBG("Parsed message: %s\n", common_chat_msgs_to_json_oaicompat<json>({msg}).at(0).dump().c_str());
     }
     return msg;

common/json-partial.cpp

@@ -254,27 +254,3 @@ bool common_json_parse(
     it = end;
     return true;
 }
-
-std::string truncate_incomplete_utf8(const std::string & str) {
-    if (str.empty()) return str;
-
-    size_t len = str.length();
-    size_t pos = len;
-
-    while (pos > 0) {
-        --pos;
-        unsigned char byte = static_cast<unsigned char>(str[pos]);
-
-        int explen;
-        if      ((byte & 0xC0) == 0x80) continue;
-        if      ((byte & 0x80) == 0x00) explen = 1; // ASCII (0xxxxxxx) - 1 byte
-        else if ((byte & 0xE0) == 0xC0) explen = 2; // 2-byte sequence (110xxxxx)
-        else if ((byte & 0xF0) == 0xE0) explen = 3; // 3-byte sequence (1110xxxx)
-        else if ((byte & 0xF8) == 0xF0) explen = 4; // 4-byte sequence (11110xxx)
-        else return str.substr(0, pos);             // Invalid UTF-8 start byte
-
-        return str.substr(0, pos + (pos + explen <= len ? explen : 0));
-    }
-
-    return "";
-}

common/json-partial.h

@@ -36,5 +36,3 @@ bool common_json_parse(
     const std::string::const_iterator & end,
     const std::string & healing_marker,
     common_json & out);
-
-std::string truncate_incomplete_utf8(const std::string & str);