feat(parser): Recover on unterminated string literals

Marwes · Marwes · commit b0422366b81e · 2020-07-08T10:05:33.000+02:00
diff --git a/base/src/error.rs b/base/src/error.rs
@@ -63,6 +63,13 @@ impl<T> Errors<T> {
     pub fn iter(&self) -> slice::Iter<T> {
         self.errors.iter()
     }
+
+    pub fn drain(
+        &mut self,
+        range: impl std::ops::RangeBounds<usize>,
+    ) -> impl Iterator<Item = T> + '_ {
+        self.errors.drain(range)
+    }
 }
 
 impl<T> Index<usize> for Errors<T> {
diff --git a/parser/src/layout.rs b/parser/src/layout.rs
@@ -130,7 +130,7 @@ where
 {
     pub fn new(tokens: Tokens) -> Layout<'input, Tokens> {
         Layout {
-            tokens: tokens,
+            tokens,
             unprocessed_tokens: Vec::new(),
             indent_levels: Contexts::new(),
         }
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
@@ -422,33 +422,17 @@ where
     Id: Clone + AsRef<str> + std::fmt::Debug,
     S: ?Sized + ParserSource,
 {
-    let layout = Layout::new(Tokenizer::new(input));
-
-    let mut parse_errors = Errors::new();
-
-    let result = grammar::TopExprParser::new().parse(
-        &input,
-        type_cache,
-        arena,
-        symbols,
-        &mut parse_errors,
-        &mut TempVecs::new(),
-        layout,
-    );
-
-    match result {
-        Ok(expr) => {
-            if parse_errors.has_errors() {
-                Err((Some(expr), transform_errors(input.span(), parse_errors)))
-            } else {
-                Ok(expr)
-            }
-        }
-        Err(err) => {
-            parse_errors.push(err);
-            Err((None, transform_errors(input.span(), parse_errors)))
-        }
-    }
+    parse_with(input, &mut |parse_errors, layout| {
+        grammar::TopExprParser::new().parse(
+            &input,
+            type_cache,
+            arena,
+            symbols,
+            parse_errors,
+            &mut TempVecs::new(),
+            layout,
+        )
+    })
 }
 
 pub fn parse_expr<'ast>(
@@ -475,34 +459,65 @@ where
     Id: Clone + Eq + Hash + AsRef<str> + ::std::fmt::Debug,
     S: ?Sized + ParserSource,
 {
-    let layout = Layout::new(Tokenizer::new(input));
+    parse_with(input, &mut |parse_errors, layout| {
+        let type_cache = TypeCache::default();
+
+        grammar::ReplLineParser::new()
+            .parse(
+                &input,
+                &type_cache,
+                arena,
+                symbols,
+                parse_errors,
+                &mut TempVecs::new(),
+                layout,
+            )
+            .map(|o| o.map(|b| *b))
+    })
+    .map_err(|(opt, err)| (opt.and_then(|opt| opt), err))
+}
+
+fn parse_with<'ast, 'input, S, T>(
+    input: &'input S,
+    parse: &mut dyn FnMut(
+        ErrorEnv<'_, 'input>,
+        Layout<'input, &mut Tokenizer<'input>>,
+    ) -> Result<
+        T,
+        lalrpop_util::ParseError<BytePos, Token<&'input str>, Spanned<Error, BytePos>>,
+    >,
+) -> Result<T, (Option<T>, ParseErrors)>
+where
+    S: ?Sized + ParserSource,
+{
+    let mut tokenizer = Tokenizer::new(input);
+    let layout = Layout::new(&mut tokenizer);
 
     let mut parse_errors = Errors::new();
 
-    let type_cache = TypeCache::default();
+    let result = parse(&mut parse_errors, layout);
+
+    let mut all_errors = transform_errors(input.span(), parse_errors);
 
-    let result = grammar::ReplLineParser::new().parse(
-        &input,
-        &type_cache,
-        arena,
-        symbols,
-        &mut parse_errors,
-        &mut TempVecs::new(),
-        layout,
-    );
+    all_errors.extend(tokenizer.errors.drain(..).map(|sp_error| {
+        pos::spanned2(
+            sp_error.span.start().absolute,
+            sp_error.span.end().absolute,
+            sp_error.value.into(),
+        )
+    }));
 
     match result {
-        Ok(repl_line) => {
-            let repl_line = repl_line.map(|b| *b);
-            if parse_errors.has_errors() {
-                Err((repl_line, transform_errors(input.span(), parse_errors)))
+        Ok(value) => {
+            if all_errors.has_errors() {
+                Err((Some(value), all_errors))
             } else {
-                Ok(repl_line)
+                Ok(value)
             }
         }
         Err(err) => {
-            parse_errors.push(err);
-            Err((None, transform_errors(input.span(), parse_errors)))
+            all_errors.push(Error::from_lalrpop(input.span(), err));
+            Err((None, all_errors))
         }
     }
 }
diff --git a/parser/src/token.rs b/parser/src/token.rs
@@ -9,6 +9,7 @@ use self::Error::*;
 use crate::{
     base::{
         ast::is_operator_byte,
+        error::Errors,
         metadata::{Comment, CommentType},
         pos::{self, BytePos, Column, Line, Location, Spanned},
     },
@@ -351,6 +352,7 @@ pub struct Tokenizer<'input> {
     input: &'input str,
     chars: CharLocations<'input>,
     start_index: BytePos,
+    pub errors: Errors<SpError>,
 }
 
 impl<'input> Tokenizer<'input> {
@@ -364,6 +366,7 @@ impl<'input> Tokenizer<'input> {
             input: input.src(),
             chars,
             start_index: input.start_index(),
+            errors: Errors::new(),
         }
     }
 
@@ -541,7 +544,16 @@ impl<'input> Tokenizer<'input> {
             }
         }
 
-        self.error(start, UnterminatedStringLiteral)
+        let end = self.chars.location;
+
+        self.errors
+            .push(pos::spanned2(start, end, UnterminatedStringLiteral));
+
+        Ok(pos::spanned2(
+            start,
+            end,
+            Token::StringLiteral(StringLiteral::Escaped(self.slice(content_start, end))),
+        ))
     }
 
     fn raw_string_literal(&mut self, start: Location) -> Result<SpannedToken<'input>, SpError> {
@@ -845,11 +857,18 @@ mod test {
     fn tokenizer<'input>(
         input: &'input str,
     ) -> impl Iterator<Item = Result<SpannedToken<'input>, SpError>> + 'input {
-        Box::new(Tokenizer::new(input).take_while(|token| match *token {
-            Ok(Spanned {
-                value: Token::EOF, ..
-            }) => false,
-            _ => true,
+        let mut tokenizer = Tokenizer::new(input);
+        Box::new(std::iter::from_fn(move || {
+            let result = tokenizer.next()?;
+            if let Some(err) = tokenizer.errors.pop() {
+                return Some(Err(err));
+            }
+            match result {
+                Ok(Spanned {
+                    value: Token::EOF, ..
+                }) => None,
+                result => Some(result),
+            }
         }))
     }
 
@@ -1019,7 +1038,11 @@ mod test {
     fn string_literal_unterminated() {
         assert_eq!(
             tokenizer(r#"foo "bar\"\n baz"#).last(),
-            Some(error(loc(4), UnterminatedStringLiteral))
+            Some(Err(pos::spanned2(
+                loc(4),
+                loc(16),
+                UnterminatedStringLiteral
+            )))
         );
     }
 
diff --git a/parser/tests/error_handling.rs b/parser/tests/error_handling.rs
@@ -69,16 +69,17 @@ y
     assert_eq!(remove_expected(result.unwrap_err().1), errors);
 }
 
-#[test]
-fn unclosed_string() {
-    let _ = ::env_logger::try_init();
-
-    let result = parse(
-        r#"
+test_parse_error! {
+    unclosed_string,
+    r#"
 "abc
-"#,
-    );
-    assert!(result.is_err());
+123"#,
+    |_: base::ast::ArenaRef<'_, '_, String>| string("abc\n123"),
+    {
+        let error = Error::Token(parser::TokenizeError::UnterminatedStringLiteral);
+        let span = pos::span(BytePos::from(0), BytePos::from(0));
+        ParseErrors::from(vec![pos::spanned(span, error)])
+    }
 }
 
 #[test]
diff --git a/parser/tests/support/mod.rs b/parser/tests/support/mod.rs
@@ -222,6 +222,10 @@ pub fn int<'a>(i: i64) -> SpExpr<'a> {
     no_loc(Expr::Literal(Literal::Int(i)))
 }
 
+pub fn string<'a>(s: &str) -> SpExpr<'a> {
+    no_loc(Expr::Literal(Literal::String(s.into())))
+}
+
 pub fn let_<'ast>(
     arena: ast::ArenaRef<'_, 'ast, String>,
     s: &str,
@@ -556,9 +560,13 @@ macro_rules! test_parse_error {
             let _ = ::env_logger::try_init();
             let text = $text;
             let result = parse(text);
-            assert!(result.is_err(), "{:#?}", result.unwrap());
+            assert!(
+                result.is_err(),
+                "Expected error but got expression: {:#?}",
+                result.unwrap()
+            );
             let (expr, err) = result.unwrap_err();
-            let expr = clear_span(expr.unwrap());
+            let expr = clear_span(expr.expect("Recovered expression"));
             mk_ast_arena!(arena);
             fn call<A, R>(a: A, f: impl FnOnce(A) -> R) -> R {
                 f(a)

Original file line number	Diff line number	Diff line change
`@@ -63,6 +63,13 @@ impl<T> Errors<T> {`
`63`	`63`	`pub fn iter(&self) -> slice::Iter<T> {`
`64`	`64`	`self.errors.iter()`
`65`	`65`	`}`
	`66`	`+`
	`67`	`+ pub fn drain(`
	`68`	`+ &mut self,`
	`69`	`+ range: impl std::ops::RangeBounds<usize>,`
	`70`	`+ ) -> impl Iterator<Item = T> + '_ {`
	`71`	`+ self.errors.drain(range)`
	`72`	`+ }`
`66`	`73`	`}`
`67`	`74`
`68`	`75`	`impl<T> Index<usize> for Errors<T> {`
Original file line number	Diff line number	Diff line change
`@@ -130,7 +130,7 @@ where`
`130`	`130`	`{`
`131`	`131`	`pub fn new(tokens: Tokens) -> Layout<'input, Tokens> {`
`132`	`132`	`Layout {`
`133`		`- tokens: tokens,`
	`133`	`+ tokens,`
`134`	`134`	`unprocessed_tokens: Vec::new(),`
`135`	`135`	`indent_levels: Contexts::new(),`
`136`	`136`	`}`