[Fix] Fix Multi-byte unicode characters in StructuralTagItem. (#396)

As reported in #392 , Multi-byte characters in StructuralTagItem `end` /
`begin` fields causes a crash. It is caused by the incorrect processing
of multi-byte characters in building grammar from StructuralTagItem, and
this mistake leads to the crash in grammar compiler. This PR fixes the
problem.


Signed-off-by: Yuchuan <[email protected]>

---------

Signed-off-by: Yuchuan <[email protected]>

Author: Seven-Streams

cpp/grammar_builder.h

@@ -95,7 +95,10 @@ class GrammarBuilder {
    * \param str The string to be added.
    */
   int32_t AddByteString(const std::string& str) {
-    std::vector<int32_t> bytes(str.begin(), str.end());
+    std::vector<int32_t> bytes;
+    for (char c : str) {
+      bytes.push_back(static_cast<int32_t>(static_cast<uint8_t>(c)));
+    }
     return AddGrammarExpr(
         {GrammarExprType::kByteString, bytes.data(), static_cast<int32_t>(bytes.size())}
     );

cpp/grammar_parser.cc

@@ -625,12 +625,7 @@ int32_t EBNFParser::ParseString() {
     return builder_.AddEmptyStr();
   }
 
-  // Convert string to bytes
-  std::vector<int32_t> bytes;
-  for (auto c : str_value) {
-    bytes.push_back(static_cast<int32_t>(static_cast<uint8_t>(c)));
-  }
-  return builder_.AddByteString(bytes);
+  return builder_.AddByteString(str_value);
 }
 
 int32_t EBNFParser::ParseRuleRef() {

tests/python/test_grammar_matcher_structural_tag.py

@@ -248,5 +248,18 @@ def test_empty_tag_dispatch():
     assert not _is_grammar_accept_string(grammar_with_stop_str, "aaa")
 
 
+@pytest.mark.hf_token_required
+def test_utf8_structural_tag_begin_end():
+    model = "deepseek-ai/DeepSeek-V3-0324"
+    tokenizer = AutoTokenizer.from_pretrained(model)
+    tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer)
+    compiler = xgr.GrammarCompiler(tokenizer_info)
+    structures = [
+        xgr.StructuralTagItem(begin="<｜tool▁calls▁begin｜>", schema={}, end="<｜tool▁calls▁end｜>")
+    ]
+    triggers = ["<｜tool▁calls▁begin｜>"]
+    _ = compiler.compile_structural_tag(structures, triggers)
+
+
 if __name__ == "__main__":
     pytest.main(sys.argv)