Support Structural Tag (#420)

This PR brings the structural tag. See the docs for its details.

It also refactors GrammarCompiler, the printing methods in EarleyParser
and GrammarMatcher, the hashing utils, the exception library.

Although introducing the new structural tag API, the API is still
backward compatible.

Signed-off-by: Ubospica <[email protected]>

---------

Signed-off-by: Ubospica <[email protected]>

Author: Ubospica

cpp/earley_parser.cc

@@ -152,7 +152,9 @@ std::pair</* scanable */ bool, /* completable */ bool> EarleyParser::Predict(
       return std::make_pair(true, false);  // The element is scanable, but not completable.
     }
     default: {
-      return std::make_pair(false, false);
+      XGRAMMAR_LOG(FATAL) << "The element type is not supported! The type is: "
+                          << int(element_expr.type);
+      XGRAMMAR_UNREACHABLE();
     }
   }
 }
@@ -178,6 +180,7 @@ void EarleyParser::Scan(const ParserState& state, const uint8_t ch) {
       default: {
         XGRAMMAR_LOG(FATAL) << "The element type is not supported! The type is: "
                             << int(element_expr.type);
+        XGRAMMAR_UNREACHABLE();
       }
     }
   } else {
@@ -269,19 +272,9 @@ void EarleyParser::PushStateAndExpand(const ParserState& state) {
   tmp_states_visited_in_queue_.Clear();
   tmp_accept_stop_token_ = false;
   tmp_states_to_be_added_.clear();
-  if (state.IsInvalid()) {
-    ExpandAndEnqueueUnexpandedState(ParserState{
-        grammar_->GetRootRuleId(),
-        ParserState::kUnexpandedRuleStartSequenceId,
-        0,
-        ParserState::kNoPrevInputPos,
-        0
-    });
-  } else {
-    // If the rule can't be expanded, we need to add it to the queue.
-    if (!ExpandAndEnqueueUnexpandedState(state)) {
-      Enqueue(state);
-    }
+  // If the rule can't be expanded, we need to add it to the queue.
+  if (!ExpandAndEnqueueUnexpandedState(state)) {
+    Enqueue(state);
   }
   rule_id_to_completable_states_.PushBack(std::vector<std::pair<int32_t, ParserState>>());
   while (!tmp_process_state_queue_.empty()) {

cpp/earley_parser.h

@@ -111,10 +111,7 @@ struct ParserState {
   }
 
   friend std::ostream& operator<<(std::ostream& os, const ParserState& state) {
-    os << "ParserState(rule_id=" << state.rule_id << ", sequence_id=" << state.sequence_id
-       << ", element_id=" << state.element_id << ", rule_start_pos=" << state.rule_start_pos
-       << ", sub_element_id=" << state.sub_element_id << ", repeat_count=" << state.repeat_count
-       << ")";
+    os << state.ToString();
     return os;
   }
 
@@ -456,10 +453,12 @@ class EarleyParser {
 
   std::string PrintStates() const {
     std::string result;
-    result += "There are " + std::to_string(scanable_state_history_.size()) + " scanable states:\n";
+    result += "There are " + std::to_string(scanable_state_history_.size()) +
+              " steps in history. Last step: [\n";
     for (const auto& state : scanable_state_history_[scanable_state_history_.size() - 1]) {
-      result += state.ToString() + "\n";
+      result += state.ToString() + ", \n";
     }
+    result += "]";
     return result;
   }
 };

cpp/fsm.h

@@ -125,21 +125,9 @@ XGRAMMAR_MEMBER_ARRAY(FSMEdge, &FSMEdge::min, &FSMEdge::max, &FSMEdge::target);
 
 }  // namespace xgrammar
 
-namespace std {
-
-/*!
- * \brief Hash function for FSMEdge.
- */
-template <>
-struct hash<xgrammar::FSMEdge> {
-  size_t operator()(const xgrammar::FSMEdge& edge) const {
-    return std::hash<std::tuple<int16_t, int16_t, int32_t>>()(
-        std::make_tuple(edge.min, edge.max, edge.target)
-    );
-  }
-};
-
-}  // namespace std
+XGRAMMAR_HASH_BY_MEMBERS(
+    xgrammar::FSMEdge, &xgrammar::FSMEdge::min, &xgrammar::FSMEdge::max, &xgrammar::FSMEdge::target
+);
 
 namespace xgrammar {
 

cpp/grammar.cc

@@ -65,11 +65,10 @@ Grammar Grammar::FromRegex(const std::string& regex, bool print_converted_ebnf)
   return FromEBNF(ebnf_string);
 }
 
-Grammar Grammar::FromStructuralTag(
-    const std::vector<StructuralTagItem>& tags, const std::vector<std::string>& triggers
+std::variant<Grammar, StructuralTagError> Grammar::FromStructuralTag(
+    const std::string& structural_tag_json
 ) {
-  Grammar grammar = StructuralTagToGrammar(tags, triggers);
-  return grammar;
+  return StructuralTagToGrammar(structural_tag_json).ToVariant();
 }
 
 // Optimized json grammar for the speed of the grammar matcher

cpp/grammar_builder.h

@@ -202,7 +202,7 @@ class GrammarBuilder {
     );
   }
 
-  int32_t AddRepeat(const int32_t ref_rule_id, int32_t min_repeat_count, int32_t max_repeat_count) {
+  int32_t AddRepeat(int32_t ref_rule_id, int32_t min_repeat_count, int32_t max_repeat_count) {
     std::vector<int32_t> data({ref_rule_id, min_repeat_count, max_repeat_count});
     return AddGrammarExpr({GrammarExprType::kRepeat, data.data(), static_cast<int32_t>(data.size())}
     );
@@ -249,6 +249,10 @@ class GrammarBuilder {
    */
   int32_t AddEmptyRule(const std::string& name) { return AddRule({name, -1}); }
 
+  int32_t AddEmptyRuleWithHint(const std::string& name_hint) {
+    return AddRule({GetNewRuleName(name_hint), -1});
+  }
+
   /*!
    * \brief Update the rule body of the given rule, specified by rule id. Can be used to set the
    * rule body of a rule inserted by GrammarBuilder::AddEmptyRule.