DSL simplification in order to propose a compilation sketch

Author: d-plaindoux

README.md

@@ -38,7 +38,6 @@ The `alter` is an annotation where:
 - `^` allows the capability to recognize negation,
 - `!` allows the capability to backtrack on failure and
 - `#` allows the capability to capture all chars.
-- `/` allows the capability to lookahead without consuming scanned elements.
 
 The `#` alteration is important because it prevents massive list construction in memory.
 
@@ -207,6 +206,18 @@ optimisations. To this end, the `AST` is translated directly into a parser
 parser using the `core` library.
 cf. [celma parser in direct style](https://github.com/d-plaindoux/celma/blob/master/lang/v0/parser/src/parser.rs).
 
+### Benchmarks
+
+Material: 
+
+```sh
+test json_apache      ... bench:   1,652,349 ns/iter (+/- 119,721) = 75 MB/s
+test json_canada_nom  ... bench:     135,309 ns/iter (+/- 4,034) = 68 MB/s
+test json_canada_pest ... bench:  61,084,354 ns/iter (+/- 2,746,834) = 36 MB/s
+test json_data        ... bench:     133,788 ns/iter (+/- 10,607) = 69 MB/s
+```
+
+
 ## V1
 
 This version targets an aggressive and an efficient parser compilation. For this

lang/v0/ast/src/syntax.rs

@@ -29,7 +29,6 @@ pub enum ASTParsec {
     PCheck(Box<ASTParsec>),
     POptional(Box<ASTParsec>),
     PRepeat(bool, Box<ASTParsec>),
-    PLookahead(Box<ASTParsec>),
 }
 
 impl ASTParsec {

lang/v0/parser/src/parser.rs

@@ -29,8 +29,8 @@ use celma_v0_core::stream::specs::Stream;
 use std::ops::Range;
 
 use celma_v0_ast::syntax::ASTParsec::{
-    PAtom, PAtoms, PBind, PCheck, PChoice, PCode, PIdent, PLookahead, PMap, PNot, POptional,
-    PRepeat, PSequence, PTry,
+    PAtom, PAtoms, PBind, PCheck, PChoice, PCode, PIdent, PMap, PNot, POptional, PRepeat,
+    PSequence, PTry,
 };
 use celma_v0_ast::syntax::{ASTParsec, ASTParsecRule};
 
@@ -204,10 +204,6 @@ where
             .and_left(skip())
             .and_right(atom2())
             .map(|p| PCheck(p.wrap())))
-        .or(a_char('/')
-            .and_left(skip())
-            .and_right(atom2())
-            .map(|p| PLookahead(p.wrap())))
         .or(atom2())
 }
 

lang/v0/parser/src/transpiler.rs

@@ -17,8 +17,8 @@
 extern crate proc_macro;
 
 use celma_v0_ast::syntax::ASTParsec::{
-    PAtom, PAtoms, PBind, PCheck, PChoice, PCode, PIdent, PLookahead, PMap, PNot, POptional,
-    PRepeat, PSequence, PTry,
+    PAtom, PAtoms, PBind, PCheck, PChoice, PCode, PIdent, PMap, PNot, POptional, PRepeat,
+    PSequence, PTry,
 };
 use celma_v0_ast::syntax::{ASTParsec, ASTParsecRule};
 use proc_macro2::{Span, TokenStream};
@@ -178,10 +178,6 @@ impl TranspileBody<(Option<String>, TokenStream)> for ASTParsec {
                     Ok((None, quote!(#pt.rep())))
                 }
             }
-            PLookahead(p) => {
-                let (_, pt) = p.transpile_body()?;
-                Ok((None, quote!(lookahead(#pt))))
-            }
         }
     }
 }

lang/v0/parser/tests/transpiler/celma_transpiler_tests.rs

@@ -76,25 +76,6 @@ mod tests_and {
         };
     }
 
-    #[test]
-    fn it_transpile_two_characters_with_lookahead_on_the_second_one() {
-        let response = celma_parsec()
-            .parse(CharStream::new("'a' /'b'"))
-            .map(|ast| ast.transpile_body());
-
-        match response {
-            Success(Ok((_, ast)), _, _) => assert_eq!(
-                ast.to_string(),
-                quote!(
-                    celma_v0_core::parser::char::a_char('a')
-                        .and_right(lookahead(celma_v0_core::parser::char::a_char('b')))
-                )
-                .to_string()
-            ),
-            _ => assert_eq!(true, false),
-        };
-    }
-
     #[test]
     fn it_transpile_two_characters_bind_left() {
         let response = celma_parsec()

lang/v1/ast/src/syntax.rs

@@ -33,6 +33,7 @@ pub enum ASTType {
 
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum ASTParsec<I> {
+    PEpsilon,
     PIdent(String),
     PAtom(I),
     PAtoms(Vec<I>),
@@ -46,7 +47,6 @@ pub enum ASTParsec<I> {
     PCheck(Box<ASTParsec<I>>),
     POptional(Box<ASTParsec<I>>),
     PRepeat(bool, Box<ASTParsec<I>>),
-    PLookahead(Box<ASTParsec<I>>),
 }
 
 impl<I> ASTParsec<I> {

lang/v1/normalizer/src/ast.rs

@@ -23,14 +23,29 @@ pub enum ASTGrammar<A> {
     Rec(String, Box<AST<A>>),
     Var(String)
 }
-
 /*
-  -- Pre-normalization
-  PN : ASTParsec -> ASTGrammar
+    PAtom(char),                    // Single char
+    PAtoms(Vec<char>),              // Char sequence
+    PBind(String, Box<ASTParsec>),  // Variable
+    PCode(String),                  // Production
+    PMap(Box<ASTParsec>, String),   // Remove?
+    PNot(Box<ASTParsec>),           // ?
+    PCheck(Box<ASTParsec>),         // No capture
+
+    -- Pre-normalization
+
+    PN : ASTParsec -> (string -> ASTParsec) -> string list -> ASTGrammar
 
-  PN[PSequence(T1,T2]] = Seq(PN[T1],PN[T2])
-  PN[PChoice(T1,T2]] = Choice(PN[T1],PN[T2])
-  PN[PRepeat(true, T)] = Rec(a,Choice(Epsilon, Seq(PN[T],String(a)) // When a is not in FV(T)
-  PN[PRepeat(false, T)] = Choice(PN[T],PN[PRepeat(true, T)])
-  PN[POptional(T)] = Choice(Epsilon, PN[T])
+                     / Var(n)               if n in l
+    PN[PIdent(n)]gl = {
+                     \ mu(n,PN[g(n)]g(n::l) otherwise
+
+    PN[PSequence(T1,T2]]gl  = Seq(PN[T1]gl,PN[T2]gl)
+    PN[PChoice(T1,T2)]gl    = Choice(PN[T1]gl,PN[T2]gl)
+    PN[PRepeat(false, T)]gl = Choice(PN[T]gl,PN[PRepeat(true, T)]gl)
+
+    PN[PRepeat(true, T)]gl  = PN[PChoice(PRepeat(false, T),PEpsilon)]gl
+    PN[POptional(T)]gl      = PN[PChoice(T,PEpsilon)]gl
+    PN[PTry(T)gl]           = PN[T]gl
  */
+

lang/v1/parser/src/parser.rs

@@ -14,13 +14,13 @@
    limitations under the License.
 */
 
-use celma_v0_core::parser::core::eos;
+use celma_v0_core::parser::core::{eos, returns, Returns};
 use celma_v0_core::parser::literal::{delimited_char, delimited_string};
 
 use celma_v0_macro::parsec_rules;
 use celma_v1_ast::syntax::ASTParsec::{
-    PAtom, PAtoms, PBind, PCheck, PChoice, PCode, PIdent, PLookahead, PMap, PNot, POptional,
-    PRepeat, PSequence, PTry,
+    PAtom, PAtoms, PBind, PCheck, PChoice, PCode, PIdent, PMap, PNot, POptional, PRepeat,
+    PSequence, PTry,
 };
 use celma_v1_ast::syntax::{ASTParsec, ASTParsecRule};
 
@@ -63,11 +63,11 @@ fn mk_ast_parsec(
         occ
     };
 
-    let add = if let Some(value) = add {
-        if value.0 {
-            PChoice(bind.wrap(), value.1.wrap())
+    let add = if let Some((choice, parser)) = add {
+        if choice {
+            PChoice(bind.wrap(), parser.wrap())
         } else {
-            PSequence(bind.wrap(), value.1.wrap())
+            PSequence(bind.wrap(), parser.wrap())
         }
     } else {
         bind
@@ -85,22 +85,25 @@ fn mk_atom(operation: Option<char>, parsec: ASTParsec<char>) -> ASTParsec<char>
         Some('^') => PNot(parsec.wrap()),
         Some('!') => PTry(parsec.wrap()),
         Some('#') => PCheck(parsec.wrap()),
-        Some('/') => PLookahead(parsec.wrap()),
         _ => parsec,
     }
 }
 
+fn epsilon() -> Returns<()> {
+    returns(())
+}
+
 parsec_rules!(
     let skip = (' '|'\t'|'\n'|'\r')* -> {}
     let ident:{String} = (skip i=#(alpha (alpha|digit|'_')*) skip) -> { i.into_iter().collect() }
 
-    let rkind = (/'>' -> {})
-              | (^('<'|'>')+ rkind -> {})
+    let rkind = (^('<'|'>')+ rkind -> {})
               | ('<' rkind '>' rkind -> {})
+              | epsilon
 
-    let rcode = (/'}' -> {})
-              | (^('}'|'{')+ rcode -> {})
+    let rcode = (^('}'|'{')+ rcode -> {})
               | ('{' rcode '}' rcode -> {})
+              | epsilon
 
     let kind:{String} = (skip '<' c=#rkind '>' skip) -> { c.into_iter().collect() }
     let code:{String} = (skip '{' c=#rcode '}' skip) -> { c.into_iter().collect() }

lang/v1/parser/tests/parser_tests.rs

@@ -23,7 +23,7 @@ mod parser_tests {
         atom_char, atom_code, atom_ident, atom_string, code, kind, parsec, rcode, rule,
     };
     use celma_v1_ast::syntax::ASTParsec::{
-        PAtom, PAtoms, PBind, PCheck, PChoice, PCode, PIdent, PLookahead, PNot, POptional,
+        PAtom, PAtoms, PBind, PCheck, PChoice, PCode, PIdent, PNot, POptional,
         PSequence, PTry,
     };
     use celma_v1_ast::syntax::ASTParsecRule;
@@ -92,7 +92,7 @@ mod parser_tests {
 
     #[test]
     fn should_parse_ident_body_with_block_unbalanced_rcode() {
-        let response = rcode().parse(CharStream::new("{ titi }"));
+        let response = rcode().parse(CharStream::new("{ titi"));
 
         assert_eq!(response.fold(|_, _, _| false, |_, _| true), true);
     }
@@ -245,23 +245,6 @@ mod parser_tests {
         );
     }
 
-    #[test]
-    fn should_parse_bind_optional_ident_body_with_lookahead() {
-        let response = parsec().parse(CharStream::new("a=/entry"));
-
-        assert_eq!(
-            response.fold(
-                |v, _, _| v
-                    == PBind(
-                        String::from("a"),
-                        PLookahead(PIdent(String::from("entry")).wrap()).wrap()
-                    ),
-                |_, _| false
-            ),
-            true
-        );
-    }
-
     #[test]
     fn should_parse_protected_simple_rule() {
         let response = rule().parse(CharStream::new("let x = entry"));