[Feature] Migrate the parsing backend to fsms. (#376)

Currently, our parser mainly accept tokens and generate masks on AST.
This PR provides a series of functions to transform ast to fsms, and
migrate the backend from AST to FSMs, which can bring better performance
in some cases. In general, this PR:
- Provide a series of functions to transform ast to fsms.
- Migrate the backend to fsms.
- Fix several bugs in fsm simlification functions.
- Refactor part of fsm functions: make them clearer.
- Refactor parser: use vectors to store completable states, instead of
maps.
- Refactor fsms: use a vector<uint8_t> to store the final states,
instead of an unordered_map.



Signed-off-by: Yuchuan <[email protected]>

Author: Seven-Streams

cpp/earley_parser.cc

@@ -7,7 +7,6 @@
 #ifndef XGRAMMAR_EARLEY_PARSER_H_
 #define XGRAMMAR_EARLEY_PARSER_H_
 #include <cstdint>
-#include <map>
 #include <ostream>
 #include <queue>
 #include <unordered_set>
@@ -219,9 +218,9 @@ class EarleyParser {
    * We divide the parser states into three categories:
    * - Scanable (which will be stored in scanable_state_history_).
    * - Predictable(If it predict a new rule successfully, then it will be stored in
-   * rule_id_to_completeable_states).
-   * - Completeable(which can perform a completion operation).
-   * A state will be stored in rule_id_to_completeable_states_ if it can be completed,
+   * rule_id_to_completable_states).
+   * - completable(which can perform a completion operation).
+   * A state will be stored in rule_id_to_completable_states_ if it can be completed,
    * and it will be stored in scanable_state_history_ if it can be scanned. Otherwise,
    * it will be discarded.
    */
@@ -238,10 +237,10 @@ class EarleyParser {
   std::vector<bool> is_completed_;
 
   /*!
-   * \brief rule_id_to_completeable_states[i][j] is the i pos j rule_id states. Earley
+   * \brief rule_id_to_completable_states[i][j] is the i pos j rule_id states. Earley
    * parser needs it to complete.
    */
-  std::vector<std::multimap<int32_t, ParserState>> rule_id_to_completeable_states_;
+  Compact2DArray<std::pair<int32_t, ParserState>> rule_id_to_completable_states_;
 
   /*!
    * \brief The states history. state_stack[i] is a vector storing the states after accepting the
@@ -261,9 +260,6 @@ class EarleyParser {
   /*! \brief The class is used to check if a state has been added into the queue. */
   RepeatDetector tmp_states_visited_in_queue_;
 
-  /*! \brief The targets of the fsm edges, used in AdvanceFsm. */
-  std::vector<int> tmp_fsm_targets_;
-
   /*! \brief Check if the stop token is accepted. */
   bool stop_token_is_accepted_ = false;
 
@@ -288,15 +284,15 @@ class EarleyParser {
    * of the grammar is used to check if the grammar is completed,
    * so it should be added into the next states.
    */
-  void Complete(const ParserState& state, const GrammarExpr& grammar_expr);
+  void Complete(const ParserState& state);
 
   /*!
    * \brief The prediction operation of the Earley parser.
    * \return First: If the state scanable, or the state is the end of the grammar,
    * then return true, otherwise return false.
    * \return Second: If the state is completable, then return true, otherwise return false.
    */
-  std::pair<bool, bool> Predict(const ParserState& state, const GrammarExpr& grammar_expr);
+  std::pair<bool, bool> Predict(const ParserState& state);
 
   /*!
    * \brief Handle the unexpanded rule, used for pushing initial state.
@@ -319,6 +315,12 @@ class EarleyParser {
       const ParserState& state, const GrammarExpr& grammar_expr, const GrammarExpr* sub_grammar_expr
   );
 
+  /*!
+   * \brief Expand the rule, used for RuleRef and kTagDispatch.
+   * \param state The state to be expanded, and it's should be on the FSM.
+   */
+  void ExpandNextRuleRefElementOnFSM(const ParserState& state);
+
   /*!
    * \brief Advance the parser to the next state, with the sub sequence is kCharacterClass.
    * \param state The state to be advanced.
@@ -359,7 +361,7 @@ class EarleyParser {
    * \param cur_sequence The sequence of the current state.
    * \return The next state, Invalid state if the character is not accepted.
    */
-  void AdvanceFsm(const ParserState& state, const uint8_t ch, const GrammarExpr& cur_sequence);
+  void AdvanceFsm(const ParserState& state, const uint8_t ch);
 
   /*!
    * \brief Enqueue the state into the queue.
@@ -373,6 +375,17 @@ class EarleyParser {
     }
   }
 
+  /*!
+   * \brief Enqueue the state into the queue, without prediction and completion.
+   * \param state The state to be enqueued.
+   */
+  void EnqueueWithoutProcessing(const ParserState& state) {
+    if (!IsStateVisitedInQueue(state)) {
+      tmp_states_visited_in_queue_.Insert(state);
+      tmp_states_to_be_added_.push_back(state);
+    }
+  }
+
  public:
   /*!
    * \brief Constructor of the Earley parser.
@@ -435,7 +448,7 @@ class EarleyParser {
    * \param state The state to be pushed.
    */
   void PushOneStateToCheck(const ParserState& state) {
-    rule_id_to_completeable_states_.emplace_back();
+    rule_id_to_completable_states_.PushBack(std::vector<std::pair<int32_t, ParserState>>());
     is_completed_.push_back(is_completed_.back());
     scanable_state_history_.PushBack(&state, 1);
     return;