feat: Accept multiple lines on incomplete repl input

Closes #830

Author: Marwes

Cargo.lock

@@ -12,7 +12,7 @@ use crate::base::{
 };
 
 use crate::{ReplLine, Variant, new_ident};
-use crate::token::{Token, StringLiteral};
+use crate::token::{Token, BorrowedToken, StringLiteral};
 use crate::ordered_float::NotNan;
 
 use crate::{Error, ErrorEnv, FieldExpr, MutIdentEnv, TempVecs, TempVecStart, Slice};
@@ -31,11 +31,11 @@ extern {
     type Location = BytePos;
     type Error = Spanned<Error, BytePos>;
 
-    enum Token<'input> {
+    enum BorrowedToken<'input> {
         "shebang line" => Token::ShebangLine(<&'input str>),
         "identifier" => Token::Identifier(<&'input str>),
         "operator" => Token::Operator(<&'input str>),
-        "string literal" => Token::StringLiteral(<StringLiteral<'input>>),
+        "string literal" => Token::StringLiteral(<StringLiteral<&'input str>>),
         "char literal" => Token::CharLiteral(<char>),
         "int literal" => Token::IntLiteral(<i64>),
         "byte literal" => Token::ByteLiteral(<u8>),
@@ -259,7 +259,7 @@ AtomicKind: ArcKind = {
                     l.into(),
                     r.into(),
                     Error::UnexpectedToken(
-                        "identifier".to_string(),
+                        Token::Identifier(id.into()),
                         ["_", "Row", "Type"].iter().map(|s| s.to_string()).collect())),
             }),
         }

parser/src/grammar.lalrpop

@@ -1,6 +1,6 @@
 use crate::base::pos::{self, BytePos, Column, Line, Location, Span, Spanned};
 
-use crate::token::{self, SpannedToken, Token};
+use crate::token::{self, BorrowedToken, SpannedToken, Token};
 
 quick_error! {
     #[derive(Debug, Eq, PartialEq, Hash, Clone)]
@@ -136,7 +136,7 @@ where
         }
     }
 
-    fn continue_block(&mut self, context: Context, token: &Token) -> Result<bool> {
+    fn continue_block(&mut self, context: Context, token: &BorrowedToken) -> Result<bool> {
         let in_rec = self.indent_levels.stack.len() >= 2
             && self.indent_levels.stack[self.indent_levels.stack.len() - 2].context == Context::Rec;
 
@@ -145,7 +145,11 @@ where
                 || (*token != Token::Rec && self.scan_continue_block(context, token)?)))
     }
 
-    fn scan_continue_block(&mut self, context: Context, first_token: &Token) -> Result<bool> {
+    fn scan_continue_block(
+        &mut self,
+        context: Context,
+        first_token: &BorrowedToken,
+    ) -> Result<bool> {
         let expected_token = match context {
             Context::Let => Token::Let,
             Context::Type => Token::Type,
@@ -218,7 +222,7 @@ where
     fn layout_token(
         &mut self,
         token: SpannedToken<'input>,
-        layout_token: Token<'input>,
+        layout_token: BorrowedToken<'input>,
     ) -> SpannedToken<'input> {
         let span = token.span;
         self.unprocessed_tokens.push(token);
@@ -600,7 +604,7 @@ where
     }
 }
 
-fn token_closes_context(token: &Token, context: Context) -> bool {
+fn token_closes_context(token: &BorrowedToken, context: Context) -> bool {
     match (token, context) {
         (&Token::Else, Context::If)
         | (&Token::RBrace, Context::Brace)
@@ -620,7 +624,7 @@ impl<'input, Tokens> Iterator for Layout<'input, Tokens>
 where
     Tokens: Iterator<Item = token::Result<SpannedToken<'input>>>,
 {
-    type Item = Result<(BytePos, Token<'input>, BytePos)>;
+    type Item = Result<(BytePos, BorrowedToken<'input>, BytePos)>;
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.layout_next_token() {

parser/src/layout.rs

@@ -27,8 +27,8 @@ use itertools::Either;
 
 use crate::base::{
     ast::{
-        self, AstType, Do, Expr, IdentEnv, PatternField, RootExpr, SpannedExpr,
-        SpannedPattern, TypedIdent, ValueBinding,
+        self, AstType, Do, Expr, IdentEnv, PatternField, RootExpr, SpannedExpr, SpannedPattern,
+        TypedIdent, ValueBinding,
     },
     error::{AsDiagnostic, Errors},
     fnv::FnvMap,
@@ -42,11 +42,12 @@ use crate::base::{
 use crate::{
     infix::{Fixity, OpMeta, OpTable, Reparser},
     layout::Layout,
-    token::{Token, Tokenizer},
+    token::{BorrowedToken, Tokenizer},
 };
 
 pub use crate::{
     infix::Error as InfixError, layout::Error as LayoutError, token::Error as TokenizeError,
+    token::Token,
 };
 
 lalrpop_mod!(
@@ -68,7 +69,7 @@ fn new_ident<Id>(type_cache: &TypeCache<Id, ArcType<Id>>, name: Id) -> TypedIden
 }
 
 type LalrpopError<'input> =
-    lalrpop_util::ParseError<BytePos, Token<'input>, Spanned<Error, BytePos>>;
+    lalrpop_util::ParseError<BytePos, BorrowedToken<'input>, Spanned<Error, BytePos>>;
 
 /// Shrink hidden spans to fit the visible expressions and flatten singleton blocks.
 fn shrink_hidden_spans<Id>(mut expr: SpannedExpr<Id>) -> SpannedExpr<Id> {
@@ -156,13 +157,13 @@ quick_error! {
         InvalidToken {
             display("Invalid token")
         }
-        UnexpectedToken(token: String, expected: Vec<String>) {
+        UnexpectedToken(token: Token<String>, expected: Vec<String>) {
             display("Unexpected token: {}{}", token, Expected(&expected))
         }
         UnexpectedEof(expected: Vec<String>) {
             display("Unexpected end of file{}", Expected(&expected))
         }
-        ExtraToken(token: String) {
+        ExtraToken(token: Token<String>) {
             display("Extra token: {}", token)
         }
         Infix(err: InfixError) {
@@ -206,7 +207,7 @@ impl Error {
                 pos::spanned2(
                     lpos,
                     rpos,
-                    Error::UnexpectedToken(token.to_string(), expected),
+                    Error::UnexpectedToken(token.map(|s| s.into()), expected),
                 )
             }
             UnrecognizedEOF {
@@ -226,7 +227,7 @@ impl Error {
             }
             ExtraToken {
                 token: (lpos, token, rpos),
-            } => pos::spanned2(lpos, rpos, Error::ExtraToken(token.to_string())),
+            } => pos::spanned2(lpos, rpos, Error::ExtraToken(token.map(|s| s.into()))),
             User { error } => error,
         }
     }

parser/src/lib.rs

@@ -16,17 +16,17 @@ use crate::{
 };
 
 #[derive(Clone, Eq, PartialEq, Hash, Debug)]
-pub enum Token<'input> {
-    ShebangLine(&'input str),
-    Identifier(&'input str),
-    Operator(&'input str),
+pub enum Token<S> {
+    ShebangLine(S),
+    Identifier(S),
+    Operator(S),
 
-    StringLiteral(StringLiteral<'input>),
+    StringLiteral(StringLiteral<S>),
     CharLiteral(char),
     IntLiteral(i64),
     ByteLiteral(u8),
     FloatLiteral(NotNan<f64>),
-    DocComment(Comment<&'input str>),
+    DocComment(Comment<S>),
 
     Rec,
     Else,
@@ -68,7 +68,10 @@ pub enum Token<'input> {
     EOF, // Required for the layout algorithm
 }
 
-impl<'input> fmt::Display for Token<'input> {
+impl<S> fmt::Display for Token<S>
+where
+    S: fmt::Display,
+{
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         use self::Token::*;
 
@@ -127,13 +130,76 @@ impl<'input> fmt::Display for Token<'input> {
     }
 }
 
+impl<S> Token<S> {
+    pub(crate) fn map<R>(self, f: impl FnOnce(S) -> R) -> Token<R> {
+        use self::Token::*;
+        match self {
+            ShebangLine(s) => ShebangLine(f(s)),
+            Identifier(s) => Identifier(f(s)),
+            Operator(s) => Operator(f(s)),
+            StringLiteral(s) => StringLiteral(match s {
+                self::StringLiteral::Escaped(s) => self::StringLiteral::Escaped(f(s)),
+                self::StringLiteral::Raw(s) => self::StringLiteral::Raw(f(s)),
+            }),
+            CharLiteral(x) => CharLiteral(x),
+            IntLiteral(x) => IntLiteral(x),
+            ByteLiteral(x) => ByteLiteral(x),
+            FloatLiteral(x) => FloatLiteral(x),
+            DocComment(Comment { typ, content }) => DocComment(Comment {
+                typ,
+                content: f(content),
+            }),
+
+            Rec => Rec,
+            Else => Else,
+            Forall => Forall,
+            If => If,
+            In => In,
+            Let => Let,
+            Do => Do,
+            Seq => Seq,
+            Match => Match,
+            Then => Then,
+            Type => Type,
+            With => With,
+
+            LBrace => LBrace,
+            LBracket => LBracket,
+            LParen => LParen,
+
+            RBrace => RBrace,
+            RBracket => RBracket,
+            RParen => RParen,
+
+            At => At,
+            Colon => Colon,
+            Comma => Comma,
+            Dot => Dot,
+            DotDot => DotDot,
+            Equals => Equals,
+            Lambda => Lambda,
+            Pipe => Pipe,
+            RArrow => RArrow,
+            Question => Question,
+
+            OpenBlock => OpenBlock,
+            CloseBlock => CloseBlock,
+            Semi => Semi,
+
+            AttributeOpen => AttributeOpen,
+
+            EOF => EOF,
+        }
+    }
+}
+
 #[derive(Clone, PartialEq, Eq, Debug, Hash)]
-pub enum StringLiteral<'input> {
-    Escaped(&'input str),
-    Raw(&'input str),
+pub enum StringLiteral<S> {
+    Escaped(S),
+    Raw(S),
 }
 
-impl StringLiteral<'_> {
+impl StringLiteral<&'_ str> {
     pub fn unescape(&self) -> String {
         match self {
             StringLiteral::Escaped(s) => unescape_string_literal(s),
@@ -164,7 +230,8 @@ fn unescape_string_literal(mut s: &str) -> String {
     string
 }
 
-pub type SpannedToken<'input> = Spanned<Token<'input>, Location>;
+pub type BorrowedToken<'input> = Token<&'input str>;
+pub type SpannedToken<'input> = Spanned<Token<&'input str>, Location>;
 
 pub type SpError = Spanned<Error, Location>;
 pub type Result<T, E = SpError> = std::result::Result<T, E>;
@@ -396,7 +463,7 @@ impl<'input> Tokenizer<'input> {
                         // FIXME: whitespace alignment
                         let doc = Token::DocComment(Comment {
                             typ: CommentType::Block,
-                            content: &comment[3..].trim(),
+                            content: comment[3..].trim(),
                         });
                         return Ok(Some(pos::spanned2(start, end, doc)));
                     } else {
@@ -786,7 +853,7 @@ mod test {
         }))
     }
 
-    fn test(input: &str, expected: Vec<(&str, Token)>) {
+    fn test(input: &str, expected: Vec<(&str, BorrowedToken<'_>)>) {
         use crate::base::source::Source;
 
         let mut tokenizer = tokenizer(input);