Merge #857

857: Recover on various tokenization errors r=Marwes a=Marwes



Co-authored-by: Markus Westerlind <[email protected]>

Author: bors[bot]

base/src/error.rs

@@ -63,6 +63,13 @@ impl<T> Errors<T> {
     pub fn iter(&self) -> slice::Iter<T> {
         self.errors.iter()
     }
+
+    pub fn drain(
+        &mut self,
+        range: impl std::ops::RangeBounds<usize>,
+    ) -> impl Iterator<Item = T> + '_ {
+        self.errors.drain(range)
+    }
 }
 
 impl<T> Index<usize> for Errors<T> {

base/tests/types.rs

@@ -155,7 +155,6 @@ fn show_record_long_type() {
         ],
         Type::string(),
     );
-    let test = data("Test", vec![data("a", vec![])]);
     let record = Type::record(
         vec![Field::new(
             "Test",

parser/src/layout.rs

@@ -130,7 +130,7 @@ where
 {
     pub fn new(tokens: Tokens) -> Layout<'input, Tokens> {
         Layout {
-            tokens: tokens,
+            tokens,
             unprocessed_tokens: Vec::new(),
             indent_levels: Contexts::new(),
         }

parser/src/lib.rs

@@ -422,33 +422,17 @@ where
     Id: Clone + AsRef<str> + std::fmt::Debug,
     S: ?Sized + ParserSource,
 {
-    let layout = Layout::new(Tokenizer::new(input));
-
-    let mut parse_errors = Errors::new();
-
-    let result = grammar::TopExprParser::new().parse(
-        &input,
-        type_cache,
-        arena,
-        symbols,
-        &mut parse_errors,
-        &mut TempVecs::new(),
-        layout,
-    );
-
-    match result {
-        Ok(expr) => {
-            if parse_errors.has_errors() {
-                Err((Some(expr), transform_errors(input.span(), parse_errors)))
-            } else {
-                Ok(expr)
-            }
-        }
-        Err(err) => {
-            parse_errors.push(err);
-            Err((None, transform_errors(input.span(), parse_errors)))
-        }
-    }
+    parse_with(input, &mut |parse_errors, layout| {
+        grammar::TopExprParser::new().parse(
+            &input,
+            type_cache,
+            arena,
+            symbols,
+            parse_errors,
+            &mut TempVecs::new(),
+            layout,
+        )
+    })
 }
 
 pub fn parse_expr<'ast>(
@@ -475,34 +459,65 @@ where
     Id: Clone + Eq + Hash + AsRef<str> + ::std::fmt::Debug,
     S: ?Sized + ParserSource,
 {
-    let layout = Layout::new(Tokenizer::new(input));
+    parse_with(input, &mut |parse_errors, layout| {
+        let type_cache = TypeCache::default();
+
+        grammar::ReplLineParser::new()
+            .parse(
+                &input,
+                &type_cache,
+                arena,
+                symbols,
+                parse_errors,
+                &mut TempVecs::new(),
+                layout,
+            )
+            .map(|o| o.map(|b| *b))
+    })
+    .map_err(|(opt, err)| (opt.and_then(|opt| opt), err))
+}
+
+fn parse_with<'ast, 'input, S, T>(
+    input: &'input S,
+    parse: &mut dyn FnMut(
+        ErrorEnv<'_, 'input>,
+        Layout<'input, &mut Tokenizer<'input>>,
+    ) -> Result<
+        T,
+        lalrpop_util::ParseError<BytePos, Token<&'input str>, Spanned<Error, BytePos>>,
+    >,
+) -> Result<T, (Option<T>, ParseErrors)>
+where
+    S: ?Sized + ParserSource,
+{
+    let mut tokenizer = Tokenizer::new(input);
+    let layout = Layout::new(&mut tokenizer);
 
     let mut parse_errors = Errors::new();
 
-    let type_cache = TypeCache::default();
+    let result = parse(&mut parse_errors, layout);
+
+    let mut all_errors = transform_errors(input.span(), parse_errors);
 
-    let result = grammar::ReplLineParser::new().parse(
-        &input,
-        &type_cache,
-        arena,
-        symbols,
-        &mut parse_errors,
-        &mut TempVecs::new(),
-        layout,
-    );
+    all_errors.extend(tokenizer.errors.drain(..).map(|sp_error| {
+        pos::spanned2(
+            sp_error.span.start().absolute,
+            sp_error.span.end().absolute,
+            sp_error.value.into(),
+        )
+    }));
 
     match result {
-        Ok(repl_line) => {
-            let repl_line = repl_line.map(|b| *b);
-            if parse_errors.has_errors() {
-                Err((repl_line, transform_errors(input.span(), parse_errors)))
+        Ok(value) => {
+            if all_errors.has_errors() {
+                Err((Some(value), all_errors))
             } else {
-                Ok(repl_line)
+                Ok(value)
             }
         }
         Err(err) => {
-            parse_errors.push(err);
-            Err((None, transform_errors(input.span(), parse_errors)))
+            all_errors.push(Error::from_lalrpop(input.span(), err));
+            Err((None, all_errors))
         }
     }
 }