Allow using generics with %parse-param

taminomara · taminomara · commit 23007f4b886a · 2025-06-06T04:10:21.000+04:00
This commit adds a new directive `%type-params`, which allows passing arbitrary generics (types and lifetimes) to the generated parser implementation.

The generics are propagated to action functions and `__GtActionsKind`, so they're available in return types as well.

This change only affects generated code: generics are not propagated to parser backend.

I've also added an example with my use case: passing an arena allocator, and returning references to AST  nodes.
diff --git a/.buildbot.sh b/.buildbot.sh
@@ -61,6 +61,9 @@ touch src/main.rs && CACHE_EXPECTED=y cargo build
 cd $root/lrpar/examples/calc_ast
 echo "2 + 3 * 4" | cargo run --package nimbleparse -- src/calc.l src/calc.y -
 echo "2 + 3 * 4" | cargo run | grep "Result: 14"
+cd $root/lrpar/examples/calc_ast_arena
+echo "2 + 3 * 4" | cargo run --package nimbleparse -- src/calc.l src/calc.y -
+echo "2 + 3 * 4" | cargo run | grep "Result: 14"
 touch src/main.rs && CACHE_EXPECTED=y cargo build
 cd $root/lrpar/examples/calc_parsetree
 echo "2 + 3 * 4" | cargo run --package nimbleparse -- src/calc.l src/calc.y -
diff --git a/Cargo.toml b/Cargo.toml
@@ -10,6 +10,7 @@ members=[
     "lrpar/examples/calc_actions",
     "lrpar/examples/calc_ast",
     "lrpar/examples/calc_parsetree",
+    "lrpar/examples/calc_ast_arena",
     "lrpar/examples/start_states",
     "lrpar/examples/clone_param",
     "lrtable",
diff --git a/cfgrammar/src/lib/yacc/ast.rs b/cfgrammar/src/lib/yacc/ast.rs
@@ -170,6 +170,7 @@ pub struct GrammarAST {
     pub expect: Option<(usize, Span)>,
     pub expectrr: Option<(usize, Span)>,
     pub parse_param: Option<(String, String)>,
+    pub parse_generics: Option<String>,
     pub programs: Option<String>,
     // The set of symbol names that, if unused in a
     // grammar, will not cause a warning or error.
@@ -248,6 +249,7 @@ impl GrammarAST {
             expect: None,
             expectrr: None,
             parse_param: None,
+            parse_generics: None,
             programs: None,
             expect_unused: Vec::new(),
         }
diff --git a/cfgrammar/src/lib/yacc/grammar.rs b/cfgrammar/src/lib/yacc/grammar.rs
@@ -83,6 +83,8 @@ pub struct YaccGrammar<StorageT = u32> {
     actions: Box<[Option<String>]>,
     /// A `(name, type)` pair defining an extra parameter to pass to action functions.
     parse_param: Option<(String, String)>,
+    /// Generic parameters (types and lifetimes) to pass to action functions.
+    parse_generics: Option<String>,
     /// Lifetimes for `param_args`
     programs: Option<String>,
     /// The actiontypes of rules (one per rule).
@@ -130,6 +132,7 @@ where
             implicit_rule: Decode::decode(decoder)?,
             actions: Decode::decode(decoder)?,
             parse_param: Decode::decode(decoder)?,
+            parse_generics: Decode::decode(decoder)?,
             programs: Decode::decode(decoder)?,
             actiontypes: Decode::decode(decoder)?,
             avoid_insert: Decode::decode(decoder)?,
@@ -168,6 +171,7 @@ where
             implicit_rule: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
             actions: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
             parse_param: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
+            parse_generics: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
             programs: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
             actiontypes: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
             avoid_insert: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
@@ -456,6 +460,7 @@ where
             implicit_rule: implicit_rule.map(|x| rule_map[&x]),
             actions: actions.into_boxed_slice(),
             parse_param: ast.parse_param.clone(),
+            parse_generics: ast.parse_generics.clone(),
             programs: ast.programs.clone(),
             avoid_insert,
             actiontypes: actiontypes.into_boxed_slice(),
@@ -628,6 +633,10 @@ where
         &self.parse_param
     }
 
+    pub fn parse_generics(&self) -> &Option<String> {
+        &self.parse_generics
+    }
+
     /// Get the programs part of the grammar
     pub fn programs(&self) -> &Option<String> {
         &self.programs
diff --git a/cfgrammar/src/lib/yacc/parser.rs b/cfgrammar/src/lib/yacc/parser.rs
@@ -549,6 +549,13 @@ impl YaccParser<'_> {
                 i = self.parse_ws(j, true)?;
                 continue;
             }
+            if let Some(j) = self.lookahead_is("%parse-generics", i) {
+                i = self.parse_ws(j, false)?;
+                let (j, ty) = self.parse_to_eol(i)?;
+                self.ast.parse_generics = Some(ty);
+                i = self.parse_ws(j, true)?;
+                continue;
+            }
             if let YaccKind::Eco = self.yacc_kind {
                 if let Some(j) = self.lookahead_is("%implicit_tokens", i) {
                     i = self.parse_ws(j, false)?;
@@ -2454,6 +2461,18 @@ x"
         );
     }
 
+    #[test]
+    fn test_parse_generics() {
+        let src = "
+          %parse-generics 'a, K, V
+          %%
+          A: 'a';
+         ";
+        let grm = parse(YaccKind::Original(YaccOriginalActionKind::UserAction), src).unwrap();
+
+        assert_eq!(grm.parse_generics, Some("'a, K, V".to_owned()));
+    }
+
     #[test]
     fn test_duplicate_rule() {
         let ast = parse(
diff --git a/doc/src/actioncode.md b/doc/src/actioncode.md
@@ -69,3 +69,19 @@ R -> ...:
   'ID' { format!("{}{}", p, ...) }
   ;
 ```
+
+# Generic parse parameter
+
+If `%parse-param` needs to be generic, additional type variables and lifetimes
+can be specified in the `%parse-generics T1, T2, ...` declaration.
+
+For example, if a grammar has following declarations:
+
+```
+%parse-generics T: FromStr
+%parse-param p: T
+```
+
+then the `parse` function will take an additional parameter of type `T`.
+
+This can be used, for example, [to allocate AST nodes in a memory arena.](https://github.com/softdevteam/grmtools/tree/master/lrpar/examples/calc_ast_arena).
diff --git a/lrpar/cttests/src/lib.rs b/lrpar/cttests/src/lib.rs
@@ -47,6 +47,9 @@ lrpar_mod!("parseparam.y");
 lrlex_mod!("parseparam_copy.l");
 lrpar_mod!("parseparam_copy.y");
 
+lrlex_mod!("typeparams.l");
+lrpar_mod!("typeparams.y");
+
 lrlex_mod!("passthrough.l");
 lrpar_mod!("passthrough.y");
 
@@ -294,6 +297,16 @@ fn test_parseparam_copy() {
     }
 }
 
+#[test]
+fn test_typeparams() {
+    let lexerdef = typeparams_l::lexerdef();
+    let lexer = lexerdef.lexer("101");
+    match typeparams_y::parse(&lexer, &3u64) {
+        (Some(104u64), _) => (),
+        _ => unreachable!(),
+    }
+}
+
 #[test]
 fn test_passthrough() {
     let lexerdef = passthrough_l::lexerdef();
diff --git a/lrpar/cttests/src/typeparams.test b/lrpar/cttests/src/typeparams.test
@@ -0,0 +1,14 @@
+name: Test %parse-param
+yacckind: Grmtools
+grammar: |
+    %start S
+    %parse-generics 'a, T: Into<u64> + Copy, R: From<u64>
+    %parse-param p: &'a T
+    %%
+    S -> R:
+        'INT' { From::from((*p).into() + $lexer.span_str($1.unwrap().span()).parse::<u64>().unwrap()) }
+    ;
+    %%
+lexer: |
+    %%
+    [0-9]+ 'INT'
diff --git a/lrpar/examples/calc_ast_arena/Cargo.toml b/lrpar/examples/calc_ast_arena/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "calc_ast_arena"
+version = "0.1.0"
+authors = ["Laurence Tratt <http://tratt.net/laurie/>"]
+edition = "2024"
+license = "Apache-2.0/MIT"
+
+[[bin]]
+doc = false
+name = "calc_ast_arena"
+
+[build-dependencies]
+cfgrammar = { path="../../../cfgrammar" }
+lrlex = { path="../../../lrlex" }
+lrpar = { path="../.." }
+
+[dependencies]
+cfgrammar = { path="../../../cfgrammar" }
+lrlex = { path="../../../lrlex" }
+lrpar = { path="../.." }
+bumpalo = "3"
diff --git a/lrpar/examples/calc_ast_arena/build.rs b/lrpar/examples/calc_ast_arena/build.rs
@@ -0,0 +1,18 @@
+#![deny(rust_2018_idioms)]
+use lrlex::CTLexerBuilder;
+
+fn main() {
+    // Since we're using both lrlex and lrpar, we use lrlex's `lrpar_config` convenience function
+    // that makes it easy to a) create a lexer and parser and b) link them together.
+    CTLexerBuilder::new()
+        .rust_edition(lrlex::RustEdition::Rust2021)
+        .lrpar_config(|ctp| {
+            ctp.rust_edition(lrpar::RustEdition::Rust2021)
+                .grammar_in_src_dir("calc.y")
+                .unwrap()
+        })
+        .lexer_in_src_dir("calc.l")
+        .unwrap()
+        .build()
+        .unwrap();
+}
diff --git a/lrpar/examples/calc_ast_arena/src/calc.l b/lrpar/examples/calc_ast_arena/src/calc.l
@@ -0,0 +1,8 @@
+%%
+[0-9]+ "INT"
+\+ "+"
+\* "*"
+\( "("
+\) ")"
+[\t\n ]+ ;
+. "UNMATCHED"
diff --git a/lrpar/examples/calc_ast_arena/src/calc.y b/lrpar/examples/calc_ast_arena/src/calc.y
@@ -0,0 +1,53 @@
+%grmtools {
+    yacckind: Grmtools,
+    test_files: "input*.txt",
+}
+%start Expr
+%avoid_insert "INT"
+%expect-unused Unmatched "UNMATCHED"
+%parse-generics 'ast
+%parse-param arena: &'ast Bump
+%%
+Expr -> Result<Expr<'ast>, ()>:
+      Expr '+' Term {
+        Ok(Expr::Add{ span: $span, lhs: arena.alloc($1?), rhs: arena.alloc($3?) })
+      }
+    | Term { $1 }
+    ;
+
+Term -> Result<Expr<'ast>, ()>:
+      Term '*' Factor {
+        Ok(Expr::Mul{ span: $span, lhs: arena.alloc($1?), rhs: arena.alloc($3?) })
+      }
+    | Factor { $1 }
+    ;
+
+Factor -> Result<Expr<'ast>, ()>:
+      '(' Expr ')' { $2 }
+    | 'INT' { Ok(Expr::Number{ span: $span }) }
+    ;
+
+Unmatched -> ():
+      "UNMATCHED" { }
+    ;
+%%
+
+use cfgrammar::Span;
+use bumpalo::Bump;
+
+#[derive(Debug)]
+pub enum Expr<'ast> {
+    Add {
+        span: Span,
+        lhs: &'ast Expr<'ast>,
+        rhs: &'ast Expr<'ast>,
+    },
+    Mul {
+        span: Span,
+        lhs: &'ast Expr<'ast>,
+        rhs: &'ast Expr<'ast>,
+    },
+    Number {
+        span: Span
+    }
+}
diff --git a/lrpar/examples/calc_ast_arena/src/input.txt b/lrpar/examples/calc_ast_arena/src/input.txt
@@ -0,0 +1 @@
+5 + 4 * 3
diff --git a/lrpar/examples/calc_ast_arena/src/main.rs b/lrpar/examples/calc_ast_arena/src/main.rs
@@ -0,0 +1,75 @@
+#![allow(clippy::unnecessary_wraps)]
+
+use std::io::{self, BufRead, Write};
+
+use cfgrammar::Span;
+use lrlex::{DefaultLexerTypes, lrlex_mod};
+use lrpar::{NonStreamingLexer, lrpar_mod};
+
+// Using `lrlex_mod!` brings the lexer for `calc.l` into scope. By default the module name will be
+// `calc_l` (i.e. the file name, minus any extensions, with a suffix of `_l`).
+lrlex_mod!("calc.l");
+// Using `lrpar_mod!` brings the parser for `calc.y` into scope. By default the module name will be
+// `calc_y` (i.e. the file name, minus any extensions, with a suffix of `_y`).
+lrpar_mod!("calc.y");
+
+use calc_y::Expr;
+
+fn main() {
+    // Get the `LexerDef` for the `calc` language.
+    let lexerdef = calc_l::lexerdef();
+    let stdin = io::stdin();
+    loop {
+        print!(">>> ");
+        io::stdout().flush().ok();
+        match stdin.lock().lines().next() {
+            Some(Ok(ref l)) => {
+                if l.trim().is_empty() {
+                    continue;
+                }
+                // Now we create a lexer with the `lexer` method with which we can lex an input.
+                let lexer = lexerdef.lexer(l);
+                let arena = bumpalo::Bump::new();
+                // Pass the lexer to the parser and lex and parse the input.
+                let (res, errs) = calc_y::parse(&lexer, &arena);
+                for e in errs {
+                    println!("{}", e.pp(&lexer, &calc_y::token_epp));
+                }
+                if let Some(Ok(r)) = res {
+                    match eval(&lexer, &r) {
+                        Ok(i) => println!("Result: {}", i),
+                        Err((span, msg)) => {
+                            let ((line, col), _) = lexer.line_col(span);
+                            eprintln!(
+                                "Evaluation error at line {} column {}, '{}' {}.",
+                                line,
+                                col,
+                                lexer.span_str(span),
+                                msg
+                            )
+                        }
+                    }
+                }
+            }
+            _ => break,
+        }
+    }
+}
+
+fn eval(
+    lexer: &dyn NonStreamingLexer<DefaultLexerTypes<u32>>,
+    e: &Expr,
+) -> Result<u64, (Span, &'static str)> {
+    match e {
+        Expr::Add { span, lhs, rhs } => eval(lexer, *lhs)?
+            .checked_add(eval(lexer, *rhs)?)
+            .ok_or((*span, "overflowed")),
+        Expr::Mul { span, lhs, rhs } => eval(lexer, *lhs)?
+            .checked_mul(eval(lexer, *rhs)?)
+            .ok_or((*span, "overflowed")),
+        Expr::Number { span } => lexer
+            .span_str(*span)
+            .parse::<u64>()
+            .map_err(|_| (*span, "cannot be represented as a u64")),
+    }
+}
diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs
