Add support to grammar for single line comments

Sometimes it would be useful in the grammar to be able to add comments
inline.  Let's allow this.

We'll treat these comments as expressions and add special handling to
allow these on lines ahead of the production name.

For now we'll render these only in the Markdown and not in the
railroad diagrams.

Author: traviscross

docs/grammar.md

@@ -27,7 +27,9 @@ BACKTICK -> U+0060
 
 LF -> U+000A
 
-Production -> `@root`? Name ` ->` Expression
+Production ->
+    ( Comment LF )*
+    `@root`? Name ` ->` Expression
 
 Name -> <Alphanumeric or `_`>+
 
@@ -55,6 +57,7 @@ Expr1 ->
       Unicode
     | NonTerminal
     | Break
+    | Comment
     | Terminal
     | Charset
     | Prose
@@ -67,6 +70,8 @@ NonTerminal -> Name
 
 Break -> LF ` `+
 
+Comment -> `//` ~[LF]+
+
 Terminal -> BACKTICK ~[LF]+ BACKTICK
 
 Charset -> `[` (` `* Characters)+ ` `* `]`
@@ -96,6 +101,7 @@ The general format is a series of productions separated by blank lines. The expr
 | Unicode | U+0060 | A single unicode character. |
 | NonTerminal | FunctionParameters | A reference to another production by name. |
 | Break | | This is used internally by the renderer to detect line breaks and indentation. |
+| Comment | // Single line comment. | A comment extending to the end of the line. |
 | Terminal | \`example\` | This is a sequence of exact characters, surrounded by backticks |
 | Charset | [ \`A\`-\`Z\` \`0\`-\`9\` \`_\` ] | A choice from a set of characters, space separated. There are three different forms. |
 | CharacterRange | [ \`A\`-\`Z\` ] | A range of characters, each character should be in backticks.

mdbook-spec/src/grammar.rs

@@ -22,6 +22,8 @@ pub struct Grammar {
 #[derive(Debug)]
 pub struct Production {
     name: String,
+    /// Comments and breaks that precede the production name.
+    comments: Vec<Expression>,
     /// Category is from the markdown lang string, and defines how it is
     /// grouped and organized on the summary page.
     category: String,
@@ -70,6 +72,8 @@ enum ExpressionKind {
     ///
     /// Used by the renderer to help format and structure the grammar.
     Break(usize),
+    /// `// Single line comment.`
+    Comment(String),
     /// ``[`A`-`Z` `_` LF]``
     Charset(Vec<Characters>),
     /// ``~[` ` LF]``
@@ -135,6 +139,7 @@ impl Expression {
             ExpressionKind::Terminal(_)
             | ExpressionKind::Prose(_)
             | ExpressionKind::Break(_)
+            | ExpressionKind::Comment(_)
             | ExpressionKind::Unicode(_) => {}
             ExpressionKind::Charset(set) => {
                 for ch in set {

mdbook-spec/src/grammar/parser.rs

@@ -122,6 +122,12 @@ impl Parser<'_> {
     }
 
     fn parse_production(&mut self, category: &str, path: &Path) -> Result<Production> {
+        let mut comments = Vec::new();
+        while let Ok(comment) = self.parse_comment() {
+            self.expect("\n", "expected newline")?;
+            comments.push(Expression::new_kind(comment));
+            comments.push(Expression::new_kind(ExpressionKind::Break(0)));
+        }
         let is_root = self.parse_is_root();
         self.space0();
         let name = self
@@ -133,6 +139,7 @@ impl Parser<'_> {
         };
         Ok(Production {
             name,
+            comments,
             category: category.to_string(),
             expression,
             path: path.to_owned(),
@@ -218,6 +225,8 @@ impl Parser<'_> {
                 bail!(self, "expected indentation on next line");
             }
             ExpressionKind::Break(space.len())
+        } else if next == b'/' {
+            self.parse_comment()?
         } else if next == b'`' {
             self.parse_terminal()?
         } else if next == b'[' {
@@ -269,6 +278,13 @@ impl Parser<'_> {
         Ok(term)
     }
 
+    /// Parse e.g. `// Single line comment.`.
+    fn parse_comment(&mut self) -> Result<ExpressionKind> {
+        self.expect("//", "expected `//`")?;
+        let text = self.take_while(&|x| x != '\n').to_string();
+        Ok(ExpressionKind::Comment(text))
+    }
+
     fn parse_charset(&mut self) -> Result<ExpressionKind> {
         self.expect("[", "expected opening [")?;
         let mut characters = Vec::new();

mdbook-spec/src/grammar/render_markdown.rs

@@ -45,6 +45,9 @@ impl Production {
             .get(&self.name)
             .map(|path| path.to_string())
             .unwrap_or_else(|| format!("missing"));
+        for expr in &self.comments {
+            expr.render_markdown(cx, output);
+        }
         write!(
             output,
             "<span class=\"grammar-text grammar-production\" id=\"{id}\" \
@@ -77,6 +80,7 @@ impl Expression {
             | ExpressionKind::Terminal(_)
             | ExpressionKind::Prose(_)
             | ExpressionKind::Break(_)
+            | ExpressionKind::Comment(_)
             | ExpressionKind::Charset(_)
             | ExpressionKind::NegExpression(_)
             | ExpressionKind::Unicode(_) => &self.kind,
@@ -163,6 +167,9 @@ impl Expression {
                 output.push_str("\\\n");
                 output.push_str(&"&nbsp;".repeat(*indent));
             }
+            ExpressionKind::Comment(s) => {
+                write!(output, "<span class=\"grammar-comment\">// {s}</span>").unwrap();
+            }
             ExpressionKind::Charset(set) => charset_render_markdown(cx, set, output),
             ExpressionKind::NegExpression(e) => {
                 output.push('~');

mdbook-spec/src/grammar/render_railroad.rs

@@ -102,8 +102,11 @@ impl Expression {
                                 .map(|e| e.render_railroad(cx, stack))
                                 .filter_map(|n| n)
                                 .collect();
+                            if seq.is_empty() {
+                                return None;
+                            }
                             let seq: Sequence<Box<dyn Node>> = Sequence::new(seq);
-                            Box::new(seq)
+                            Some(Box::new(seq))
                         };
 
                         // If `stack` is true, split the sequence on Breaks and
@@ -127,16 +130,18 @@ impl Expression {
                                 &es[..]
                             };
 
-                            let mut breaks: Vec<_> =
-                                es.split(|e| e.is_break()).map(|es| make_seq(es)).collect();
+                            let mut breaks: Vec<_> = es
+                                .split(|e| e.is_break())
+                                .flat_map(|es| make_seq(es))
+                                .collect();
                             // If there aren't any breaks, don't bother stacking.
-                            if breaks.len() == 1 {
-                                breaks.pop().unwrap()
-                            } else {
-                                Box::new(Stack::new(breaks))
+                            match breaks.len() {
+                                0 => return None,
+                                1 => breaks.pop().unwrap(),
+                                _ => Box::new(Stack::new(breaks)),
                             }
                         } else {
-                            make_seq(&es)
+                            make_seq(&es)?
                         }
                     }
                     // Treat `e?` and `e{..1}` / `e{0..1}` equally.
@@ -205,6 +210,7 @@ impl Expression {
                     ExpressionKind::Terminal(t) => Box::new(Terminal::new(t.clone())),
                     ExpressionKind::Prose(s) => Box::new(Terminal::new(s.clone())),
                     ExpressionKind::Break(_) => return None,
+                    ExpressionKind::Comment(_) => return None,
                     ExpressionKind::Charset(set) => {
                         let ns: Vec<_> = set.iter().map(|c| c.render_railroad(cx)).collect();
                         Box::new(Choice::<Box<dyn Node>>::new(ns))

src/notation.md

@@ -27,6 +27,7 @@ The following notations are used by the *Lexer* and *Syntax* grammar snippets:
 | U+xxxx            | U+0060                        | A single unicode character                |
 | \<text\>          | \<any ASCII char except CR\>  | An English description of what should be matched |
 | Rule <sub>suffix</sub> | IDENTIFIER_OR_KEYWORD <sub>_except `crate`_</sub> | A modification to the previous rule |
+| // Comment. | // Single line comment. | A comment extending to the end of the line. |
 
 Sequences have a higher precedence than `|` alternation.
 

theme/reference.css

@@ -24,6 +24,10 @@
     --alert-example-color: #8250df;
     --grammar-literal-bg: #dedede;
 }
+.light, .rust {
+    --grammar-comment-color: lch(from var(--quote-bg) calc(l - 50) 0 0);
+    --inline-code-color: var(--grammar-comment-color);
+}
 .coal, .navy {
     --alert-note-color: #4493f8;
     --alert-warning-color: #d29922;
@@ -39,6 +43,8 @@
     --gramar-literal-bg: #191f26;
 }
 .coal, .navy, .ayu {
+    --grammar-comment-color: lch(from var(--quote-bg) calc(l + 50) 0 0);
+    --inline-code-color: var(--grammar-comment-color);
     --railroad-background-color: hsl(230, 10%, 20%);
     --railroad-background-image:
         linear-gradient(to right, rgba(150, 150, 150, .05) 1px, transparent 1px),
@@ -609,6 +615,15 @@ main > .rule {
     font-family: "Open Sans", sans-serif;
 }
 
+/* Comments inside the grammar. */
+.grammar-comment {
+    font-family: "Open Sans", sans-serif;
+    color: var(--grammar-comment-color);
+}
+.grammar-comment code.hljs {
+    background: var(--grammar-literal-bg);
+}
+
 /* Places a box around literals to differentiate from other grammar punctuation like | and ( . */
 .grammar-literal {
     font-family: var(--mono-font);