Merge pull request #498 from ratmice/lrlex_allow_wholeline_comments

Add allow_wholeline_comments LexFlag.

Author: ltratt

lrlex/src/lib/ctbuilder.rs

@@ -482,6 +482,7 @@ pub fn lexerdef() -> {lexerdef_type} {{
         .ok();
 
         let LexFlags {
+            allow_wholeline_comments,
             dot_matches_new_line,
             multi_line,
             octal,
@@ -494,6 +495,7 @@ pub fn lexerdef() -> {lexerdef_type} {{
             dfa_size_limit,
             nest_limit,
         } = lex_flags;
+        let allow_wholeline_comments = QuoteOption(allow_wholeline_comments);
         let dot_matches_new_line = QuoteOption(dot_matches_new_line);
         let multi_line = QuoteOption(multi_line);
         let octal = QuoteOption(octal);
@@ -508,6 +510,7 @@ pub fn lexerdef() -> {lexerdef_type} {{
 
         outs.push_str(&format!(
             "let mut lex_flags = ::lrlex::DEFAULT_LEX_FLAGS;
+            lex_flags.allow_wholeline_comments = {allow_wholeline_comments};
             lex_flags.dot_matches_new_line = {dot_matches_new_line};
             lex_flags.multi_line = {multi_line};
             lex_flags.octal = {octal};
@@ -521,6 +524,7 @@ pub fn lexerdef() -> {lexerdef_type} {{
             lex_flags.nest_limit = {nest_limit};
             let lex_flags = lex_flags;
 ",
+            allow_wholeline_comments = quote!(#allow_wholeline_comments),
             dot_matches_new_line = quote!(#dot_matches_new_line),
             multi_line = quote!(#multi_line),
             octal = quote!(#octal),

lrlex/src/lib/lexer.rs

@@ -28,6 +28,7 @@ pub struct LexFlags {
     pub multi_line: Option<bool>,
     pub octal: Option<bool>,
     pub posix_escapes: Option<bool>,
+    pub allow_wholeline_comments: Option<bool>,
 
     // All the following values when `None` default to the `regex` crate's default value.
     pub case_insensitive: Option<bool>,
@@ -43,6 +44,9 @@ impl LexFlags {
     /// Merges flags from `other` into `self`
     /// Flags which are `Some` in `other` overriding flags in self.
     pub fn merge_from(&mut self, other: &Self) {
+        if other.allow_wholeline_comments.is_some() {
+            self.allow_wholeline_comments = other.allow_wholeline_comments;
+        }
         if other.dot_matches_new_line.is_some() {
             self.dot_matches_new_line = other.dot_matches_new_line;
         }
@@ -81,6 +85,7 @@ impl LexFlags {
 
 /// LexFlags with flags set to default values.
 pub const DEFAULT_LEX_FLAGS: LexFlags = LexFlags {
+    allow_wholeline_comments: Some(false),
     dot_matches_new_line: Some(true),
     multi_line: Some(true),
     octal: Some(true),
@@ -96,6 +101,7 @@ pub const DEFAULT_LEX_FLAGS: LexFlags = LexFlags {
 
 /// LexFlags with all of the values `None`.
 pub const UNSPECIFIED_LEX_FLAGS: LexFlags = LexFlags {
+    allow_wholeline_comments: None,
     dot_matches_new_line: None,
     multi_line: None,
     octal: None,

lrlex/src/lib/parser.rs

@@ -240,7 +240,8 @@ where
         span_map: &mut HashMap<&str, Span>,
         lex_flags: &mut LexFlags,
     ) -> LexInternalBuildResult<usize> {
-        const OPTIONS: [&str; 11] = [
+        const OPTIONS: [&str; 12] = [
+            "allow_wholeline_comments",
             "dot_matches_new_line",
             "multi_line",
             "octal",
@@ -273,6 +274,7 @@ where
                     });
                 }
                 match opt {
+                    "allow_wholeline_comments" => lex_flags.allow_wholeline_comments = Some(flag),
                     "case_insensitive" => lex_flags.case_insensitive = Some(flag),
                     "swap_greed" => lex_flags.swap_greed = Some(flag),
                     "ignore_whitespace" => lex_flags.ignore_whitespace = Some(flag),
@@ -350,6 +352,15 @@ where
         self.lex_flags.merge_from(&self.default_lex_flags);
         loop {
             i = self.parse_ws(i)?;
+            if self.lex_flags.allow_wholeline_comments.unwrap_or(false)
+                && self.lookahead_is("//", i).is_some()
+            {
+                i = RE_LINE_SEP
+                    .find(&self.src[i..])
+                    .map(|m| m.start() + i)
+                    .unwrap_or(self.src.len());
+                continue;
+            }
             if i == self.src.len() {
                 break Err(self.mk_error(LexErrorKind::PrematureEnd, i));
             }
@@ -470,6 +481,16 @@ where
             // We should be at newline of the previous section separator '%%<here>\n upon entry,
             // otherwise after iterating before the newline of the previous iterations rule or at eof.
             i = self.parse_nl(i)?;
+            let line_len = RE_LINE_SEP
+                .find(&self.src[i..])
+                .map(|m| m.start())
+                .unwrap_or(self.src.len() - i);
+            if self.lex_flags.allow_wholeline_comments.unwrap_or(false)
+                && self.lookahead_is("//", i).is_some()
+            {
+                i += line_len;
+                continue;
+            }
             // According to posix lex:
             //
             // > Any such input (beginning with a <blank> or within "%{" and "%}" delimiter lines)
@@ -486,10 +507,6 @@ where
             // Previously we allowed these, and trimmed leading spaces, parsing any rules after them. Currently we will emit an error.
             let j = self.parse_ws(i)?;
             if j != i {
-                let line_len = RE_LINE_SEP
-                    .find(&self.src[i..])
-                    .map(|m| m.start())
-                    .unwrap_or(self.src.len() - i);
                 let err = LexBuildError {
                     kind: LexErrorKind::VerbatimNotSupported,
                     spans: vec![Span::new(i, i + line_len)],
@@ -1876,4 +1893,41 @@ b "A"
             3,
         );
     }
+
+    #[test]
+    fn test_comments() {
+        let src = r#"
+%grmtools {allow_wholeline_comments}
+// comment
+%s InclusiveState
+%%
+// "comment but an invalid rule name if parsed as a rule"
+\/\/ 'escaping_required'
+. 'dot'
+"#;
+        LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(src).unwrap();
+        let src = r#"
+// comments not allowed.
+%s InclusiveState
+%%
+. 'dot'
+"#;
+        LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(src).expect_error_at_line_col(
+            src,
+            LexErrorKind::UnknownDeclaration,
+            2,
+            1,
+        );
+        let src = r#"
+%%
+// "Invalid rule name"
+. 'dot'
+"#;
+        LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(src).expect_error_at_line_col(
+            src,
+            LexErrorKind::InvalidName,
+            3,
+            18,
+        );
+    }
 }