Lex block comments

Author: Kmeakin

fathom/src/surface/lexer.rs

@@ -1,5 +1,5 @@
 use codespan_reporting::diagnostic::{Diagnostic, Label};
-use logos::Logos;
+use logos::{Filter, Logos};
 
 use crate::{
     files::FileId,
@@ -15,6 +15,7 @@ pub fn is_keyword(word: &str) -> bool {
 }
 
 #[derive(Clone, Debug, Logos)]
+#[logos(extras = FileId)]
 pub enum Token<'source> {
     #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*")]
     #[regex(r"r#[a-zA-Z_][a-zA-Z0-9_]*", |lex| &lex.slice()[2..])]
@@ -107,23 +108,78 @@ pub enum Token<'source> {
     #[token(")")]
     CloseParen,
 
+    #[token(r"/*", block_comment)]
+    BlockComment(BlockCommentError),
+
     #[error]
     #[regex(r"\p{Whitespace}", logos::skip)]
     #[regex(r"//(.*)\n", logos::skip)]
     Error,
 }
 
+const OPEN: &str = "/*";
+const CLOSE: &str = "*/";
+const LEN: BytePos = OPEN.len() as BytePos;
+
+fn block_comment<'source>(
+    lexer: &mut logos::Lexer<'source, Token<'source>>,
+) -> Filter<BlockCommentError> {
+    let start = lexer.span().start as BytePos;
+    let first_open_pos = start;
+    let mut last_close_pos = start;
+    let mut pos = start;
+
+    let mut depth: u32 = 1;
+    while let Some(c) = lexer.remainder().chars().next() {
+        if lexer.remainder().starts_with(OPEN) {
+            pos += LEN;
+            lexer.bump(OPEN.len());
+            depth += 1;
+        } else if lexer.remainder().starts_with(CLOSE) {
+            pos += LEN;
+            last_close_pos = pos;
+            lexer.bump(CLOSE.len());
+            depth -= 1;
+            if depth == 0 {
+                break;
+            }
+        } else {
+            pos += c.len_utf8() as BytePos;
+            lexer.bump(c.len_utf8());
+        }
+    }
+
+    let file_id = lexer.extras;
+    match depth {
+        0 => Filter::Skip,
+        _ => Filter::Emit(BlockCommentError {
+            depth,
+            first_open: ByteRange::new(file_id, first_open_pos, first_open_pos + LEN),
+            last_close: ByteRange::new(file_id, last_close_pos, last_close_pos + LEN),
+        }),
+    }
+}
+
 pub type Spanned<Tok, Loc> = (Loc, Tok, Loc);
 
 #[derive(Clone, Debug)]
 pub enum Error {
+    UnclosedBlockComment(BlockCommentError),
     UnexpectedCharacter { range: ByteRange },
 }
 
+#[derive(Clone, Debug)]
+pub struct BlockCommentError {
+    depth: u32,
+    first_open: ByteRange,
+    last_close: ByteRange,
+}
+
 impl Error {
     pub fn range(&self) -> ByteRange {
         match self {
             Error::UnexpectedCharacter { range } => *range,
+            Error::UnclosedBlockComment(BlockCommentError { first_open, .. }) => *first_open,
         }
     }
 
@@ -132,6 +188,19 @@ impl Error {
             Error::UnexpectedCharacter { range } => Diagnostic::error()
                 .with_message("unexpected character")
                 .with_labels(vec![Label::primary(range.file_id(), *range)]),
+            Error::UnclosedBlockComment(BlockCommentError {
+                depth,
+                first_open,
+                last_close,
+            }) => Diagnostic::error()
+                .with_message("unclosed block comment")
+                .with_labels(vec![
+                    Label::primary(first_open.file_id(), *first_open)
+                        .with_message(format!("first `{OPEN}`")),
+                    Label::primary(last_close.file_id(), *last_close)
+                        .with_message(format!("last `{CLOSE}`")),
+                ])
+                .with_notes(vec![format!("Help: {depth} more `{CLOSE}` needed",)]),
         }
     }
 }
@@ -145,16 +214,19 @@ pub fn tokens(
         "`source` must be less than 4GiB in length"
     );
 
-    Token::lexer(source).spanned().map(move |(token, range)| {
-        let start = range.start as BytePos;
-        let end = range.end as BytePos;
-        match token {
-            Token::Error => Err(Error::UnexpectedCharacter {
-                range: ByteRange::new(file_id, start, end),
-            }),
-            token => Ok((start, token, end)),
-        }
-    })
+    Token::lexer_with_extras(source, file_id)
+        .spanned()
+        .map(move |(token, range)| {
+            let start = range.start as BytePos;
+            let end = range.end as BytePos;
+            match token {
+                Token::BlockComment(err) => Err(Error::UnclosedBlockComment(err)),
+                Token::Error => Err(Error::UnexpectedCharacter {
+                    range: ByteRange::new(file_id, start, end),
+                }),
+                token => Ok((start, token, end)),
+            }
+        })
 }
 
 impl<'source> Token<'source> {
@@ -197,6 +269,7 @@ impl<'source> Token<'source> {
             Token::CloseBracket => "]",
             Token::OpenParen => "(",
             Token::CloseParen => ")",
+            Token::BlockComment(_) => "block comment",
             Token::Error => "error",
             Token::BangEquals => "!=",
             Token::EqualsEquals => "==",

tests/fail/elaboration/block-comment.fathom

@@ -0,0 +1,2 @@
+let x = /* true */ false;
+{}

tests/fail/elaboration/block-comment.snap

@@ -0,0 +1,4 @@
+stdout = '''
+let x : Bool = false; () : ()
+'''
+stderr = ''

tests/fail/parse/unclosed-block-comment.fathom

@@ -0,0 +1,10 @@
+//~ exit-code = 1
+
+{}
+/* a
+/* b
+/* c
+
+*/ c
+*/ b
+// a

tests/fail/parse/unclosed-block-comment.snap

@@ -0,0 +1,14 @@
+stdout = ''
+stderr = '''
+error: unclosed block comment
+  ┌─ tests/fail/parse/unclosed-block-comment.fathom:4:1
+  │
+4 │ /* a
+  │ ^^ first `/*`
+  ·
+9 │ */ b
+  │ ^^ last `*/`
+  │
+  = Help: 1 more `*/` needed
+
+'''