Improve parsing of newlines where possibly indented blocks are expected

Author: LPTK

hkmc2/shared/src/main/scala/hkmc2/semantics/Elaborator.scala

@@ -623,7 +623,7 @@ extends Importer:
         case Block(sts) :: trees =>
           go(acc, sts ::: trees)
         case tree :: trees =>
-          raise(ErrorReport(msg"Illegal juxtaposition right-hand side." -> tree.toLoc :: Nil))
+          raise(ErrorReport(msg"Illegal juxtaposition right-hand side (${tree.describe})." -> tree.toLoc :: Nil))
           go(acc, trees)
 
       go(term(lhs), rhs :: Nil)

hkmc2/shared/src/main/scala/hkmc2/syntax/Parser.scala

@@ -240,6 +240,17 @@ abstract class Parser(
 
   final def maybeIndented[R](f: (Parser, Bool) => R): R =
     yeetSpaces match
+      case (NEWLINE, l0) :: _ =>
+        consume
+        while yeetSpaces.headOption.exists(_._1 === NEWLINE) do consume
+        cur match
+        case Nil =>
+        case (tok, loc) :: _ =>
+          raise(WarningReport(
+            msg"This ${tok.describe} should be indented" -> S(loc) :: 
+            msg"since it is a continuation of the new line here" -> S(l0) :: 
+            Nil))
+        maybeIndented(f)
       case (br @ BRACKETS(Indent | Curly, toks), _) :: _ =>
         consume
         rec(toks, S(br.innerLoc), br.describe).concludeWith(f(_, true))
@@ -278,7 +289,7 @@ abstract class Parser(
             consume
             val blk = rec(toks, S(tok.innerLoc), tok.describe).concludeWith(_.blockOf(subRule, Nil, allowNewlines)) // FIXME allowNewlines?
             if blk.isEmpty then
-              err((msg"Expected ${subRule.whatComesAfter} ${subRule.mkAfterStr}; found end of block instead" -> S(loc) :: Nil))
+              err(msg"Expected ${subRule.whatComesAfter} ${subRule.mkAfterStr}; found end of block instead" -> S(loc) :: Nil)
               errExpr
             blk.map(annotations.annotate) ::: blockContOf(rule)
           case _ =>
@@ -307,10 +318,10 @@ abstract class Parser(
                 annotations.annotate(parseRule(CommaPrecNext, exprAlt.rest).map(res => exprAlt.k(e, res)).getOrElse(errExpr)) :: blockContOf(rule)
               case N =>
                 // TODO dedup?
-                err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(loc) :: Nil))
+                err(msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(loc) :: Nil)
                 annotations.annotate(errExpr) :: blockContOf(rule)
           case N =>
-            err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(loc) :: Nil))
+            err(msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(loc) :: Nil)
             annotations.annotate(errExpr) :: blockContOf(rule)
       case N =>
         val lhs = tryParseExp(CommaPrecNext, tok, loc, rule).getOrElse(errExpr)
@@ -336,7 +347,7 @@ abstract class Parser(
         case S(res) =>
           S(res)
         case N =>
-          err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(loc) :: Nil))
+          err(msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(loc) :: Nil)
           N
 
 
@@ -348,7 +359,7 @@ abstract class Parser(
       case S(res) => S(res)
       case N =>
         consume
-        err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(loc) :: Nil))
+        err(msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(loc) :: Nil)
         N
     yeetSpaces match
     // case (tok @ (id: IDENT), loc) :: _ if Keyword.all.get(id.name).exists(_.leftPrecOrMin < prec) =>
@@ -358,7 +369,7 @@ abstract class Parser(
     //     case S(res) =>
     //       S(res)
     //     case N =>
-    //       err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found end of phrase instead" -> S(loc.left) :: Nil))
+    //      err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found end of phrase instead" -> S(loc.left) :: Nil)
     //       N
     case (tok @ (id: IDENT), loc) :: _ =>
       Keyword.all.get(id.name) match
@@ -397,14 +408,14 @@ abstract class Parser(
       rule.emptyAlt match
         case S(res) => S(res)
         case N =>
-          // err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> lastLoc :: Nil))
-          err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(l0) :: Nil))
+          //err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> lastLoc :: Nil)
+          err(msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> S(l0) :: Nil)
           N
     case (br @ BRACKETS(Indent | Curly, toks), loc) :: _ =>
       // rule.blkAlt match
       //   case S(res) => S(res)
       //   case N =>
-      //     err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> lastLoc :: Nil))
+      //    err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${tok.describe} instead" -> lastLoc :: Nil)
       //     N
 
       if verbose then printDbg("$ found an indented" + (toks match
@@ -445,7 +456,7 @@ abstract class Parser(
         case S(res) =>
           S(res)
         case N =>
-          err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found end of input instead" -> lastLoc :: Nil))
+          err(msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found end of input instead" -> lastLoc :: Nil)
           N
 
 
@@ -462,7 +473,7 @@ abstract class Parser(
         yeetSpaces match
           case (IDENT("=", _), l1) :: _ => consume
           case (tk, l1) :: _ =>
-            err((msg"Expected `=` after ${nme}; found ${tk.toString} instead" -> S(l1) :: Nil))
+            err(msg"Expected `=` after ${nme}; found ${tk.toString} instead" -> S(l1) :: Nil)
         val rhs = simpleExprImpl(0)
         val v = Tree.Ident(nme).withLoc(S(l0))
         cur match {
@@ -495,7 +506,7 @@ abstract class Parser(
       Keyword.all.get(nme) match
         case S(kw) => // * Expressions starting with keywords should be handled in parseRule
           // * I guess this case is not really supposed to be ever reached (?)
-          err((msg"Unexpected ${kw.toString} in this position" -> S(loc) :: Nil))
+          err(msg"Unexpected ${kw.toString} in this position" -> S(loc) :: Nil)
           errExpr
         case N =>
           consume
@@ -554,7 +565,7 @@ abstract class Parser(
               consume
               simpleExpr(0)
             case (tk, loc) :: _ =>
-              err((msg"Expected an expression; found ${tk.toString} instead" -> S(loc) :: Nil))
+              err(msg"Expected an expression; found ${tk.toString} instead" -> S(loc) :: Nil)
               errExpr
             case Nil =>
               err(msg"Expected '`in'; found end of input instead" -> lastLoc :: Nil)
@@ -578,7 +589,7 @@ abstract class Parser(
                   err(msg"Expected '`in'; found ${tk.toString} instead" -> tk.toLoc :: Nil)
                   errExpr
             case (tk, loc) :: _ =>
-              err((msg"Expected 'else'; found ${tk.toString} instead" -> S(loc) :: Nil))
+              err(msg"Expected 'else'; found ${tk.toString} instead" -> S(loc) :: Nil)
               errExpr
             case Nil =>
               err(msg"Expected 'else'; found end of input instead" -> lastLoc :: Nil)
@@ -594,19 +605,23 @@ abstract class Parser(
         case _ => unsupportedQuote(S(loc))
       }
     case (BRACKETS(Indent | Curly, _), loc) :: _ =>
-      err((msg"Expected an expression; found block instead" -> lastLoc :: Nil))
+      err(msg"Expected an expression; found block instead" -> lastLoc :: Nil)
       errExpr
     case (SUSPENSION(dotDotDot), loc) :: _ =>
       consume
       val bod = yeetSpaces match
         case Nil | (COMMA, _) :: _ => N
         case _ => S(expr(prec))
       Spread(if dotDotDot then Keyword.`...` else Keyword.`..`, S(loc), bod)
+    // case (NEWLINE, loc) :: _ => // this seems to never be reached
+    //   raise(WarningReport(msg"???" -> S(loc) :: Nil))
+    //   consume
+    //   simpleExprImpl(prec)
     case (tok, loc) :: _ =>
-      err((msg"Expected an expression; found new line instead" -> S(loc) :: Nil))
+      err(msg"Expected an expression; found ${tok.describe} instead" -> S(loc) :: Nil)
       errExpr
     case Nil =>
-      err((msg"Expected an expression; found end of input instead" -> lastLoc :: Nil))
+      err(msg"Expected an expression; found end of input instead" -> lastLoc :: Nil)
       errExpr
 
 
@@ -727,7 +742,7 @@ abstract class Parser(
       }
       case (COMMA, l0) :: _ if prec === 0 =>
         consume
-        err((msg"Unexpected comma in this position" -> S(l0) :: Nil))
+        err(msg"Unexpected comma in this position" -> S(l0) :: Nil)
         acc
         /* 
       case (KEYWORD(opStr @ "=>"), l0) :: (NEWLINE, l1) :: _ if opPrec(opStr)._1 > prec =>
@@ -986,7 +1001,7 @@ abstract class Parser(
                 .getOrElse(errExpr)
               case N =>
                 // TODO other alts...?
-                err((msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${kw.name} instead" -> S(l0) :: Nil))
+                err(msg"Expected ${rule.whatComesAfter} ${rule.mkAfterStr}; found ${kw.name} instead" -> S(l0) :: Nil)
                 acc
           case _ => acc
       case _ =>

hkmc2/shared/src/main/scala/hkmc2/syntax/Token.scala

@@ -10,7 +10,7 @@ sealed abstract class Token:
     case SPACE => "space"
     case COMMA => "comma"
     case SEMI => "semicolon"
-    case NEWLINE => "newline"
+    case NEWLINE => "new line"
     case INDENT => "indentation"
     case DEINDENT => "deindentation"
     case ERROR => "error"

hkmc2/shared/src/main/scala/hkmc2/syntax/Tree.scala

@@ -133,8 +133,9 @@ enum Tree extends AutoLocated:
     case IntLit(value) => "integer literal"
     case DecLit(value) => "decimal literal"
     case StrLit(value) => "string literal"
-    case UnitLit(value) => if value then "null" else "undefined"
     case BoolLit(value) => s"$value literal"
+    case UnitLit(value) => if value then "null" else "undefined"
+    case Unt() => "unit"
     case Bra(k, _) => k.name + " section"
     case Block(stmts) => "block"
     case OpBlock(_) => "operator block"

hkmc2/shared/src/test/mlscript/backlog/ToTriage.mls

@@ -19,7 +19,7 @@
 "a" +
 // a
 "b"
-//│ ╔══[PARSE ERROR] Expected start of statement in this position; found newline instead
+//│ ╔══[PARSE ERROR] Expected start of statement in this position; found new line instead
 //│ ║  l.20: 	// a
 //│ ║        	    ^
 //│ ║  l.21: 	"b"
@@ -276,7 +276,7 @@ class Foo' with
 module Example with
 // whoops
   val a = this
-//│ ╔══[PARSE ERROR] Expected block after type declaration body; found newline instead
+//│ ╔══[PARSE ERROR] Expected block after type declaration body; found new line instead
 //│ ║  l.276: 	module Example with
 //│ ║         	                   ^
 //│ ║  l.277: 	// whoops

hkmc2/shared/src/test/mlscript/basics/BlockArgs.mls

@@ -11,7 +11,7 @@ foo(1){x => x}
 //│ ╔══[ERROR] Name not found: foo
 //│ ║  l.10: 	foo(1){x => x}
 //│ ╙──      	^^^
-//│ ╔══[ERROR] Illegal juxtaposition right-hand side.
+//│ ╔══[ERROR] Illegal juxtaposition right-hand side (infix operation).
 //│ ║  l.10: 	foo(1){x => x}
 //│ ╙──      	       ^^^^^^
 
@@ -20,15 +20,15 @@ foo(1)
 //│ ╔══[ERROR] Name not found: foo
 //│ ║  l.18: 	foo(1)
 //│ ╙──      	^^^
-//│ ╔══[ERROR] Illegal juxtaposition right-hand side.
+//│ ╔══[ERROR] Illegal juxtaposition right-hand side (infix operation).
 //│ ║  l.19: 	  x => x
 //│ ╙──      	  ^^^^^^
 
 foo(1) { x => x }
 //│ ╔══[ERROR] Name not found: foo
 //│ ║  l.27: 	foo(1) { x => x }
 //│ ╙──      	^^^
-//│ ╔══[ERROR] Illegal juxtaposition right-hand side.
+//│ ╔══[ERROR] Illegal juxtaposition right-hand side (infix operation).
 //│ ║  l.27: 	foo(1) { x => x }
 //│ ╙──      	         ^^^^^^
 

hkmc2/shared/src/test/mlscript/basics/Indentation.mls

@@ -32,14 +32,54 @@ tuple(
 tuple()
 //│ = []
 
-:fixme
 tuple(
+  id(2
+    +2))
+    tuple()
+  tuple()
+//│ = [[[4]]]
+
+// 🤔
+:e
+tuple(
+  id(2
+    +2))
+      tuple()
+    tuple()
+  tuple()
+//│ ╔══[ERROR] Illegal juxtaposition right-hand side (juxtaposition).
+//│ ║  l.47: 	      tuple()
+//│ ║        	      ^^^^^^^
+//│ ║  l.48: 	    tuple()
+//│ ║        	^^^^^^^^^^^
+//│ ║  l.49: 	  tuple()
+//│ ╙──      	^^
+//│ = [[4]]
+
+module A with {
+}
+
+:p
+tuple(
+)
+//│ |tuple|(|↵|)|
+//│ Parsed:
+//│ 	App(Ident(tuple),Tup(List()))
+//│ = []
+
+:w
+tuple(
+1
 )
-//│ ╔══[PARSE ERROR] Expected an expression; found new line instead
-//│ ║  l.36: 	tuple(
+//│ ╔══[WARNING] This literal should be indented
+//│ ║  l.72: 	1
+//│ ║        	^
+//│ ╟── since it is a continuation of the new line here
+//│ ║  l.71: 	tuple(
 //│ ║        	      ^
-//│ ║  l.37: 	)
+//│ ║  l.72: 	1
 //│ ╙──      	
+//│ = [1]
 
 tuple(
   )
@@ -98,9 +138,9 @@ id
 //│ |id|→|1|←|
 //│ Parsed:
 //│ 	Jux(Ident(id),Block(List(IntLit(1))))
-//│ ╔══[ERROR] Illegal juxtaposition right-hand side.
-//│ ║  l.97: 	  1
-//│ ╙──      	  ^
+//│ ╔══[ERROR] Illegal juxtaposition right-hand side (integer literal).
+//│ ║  l.137: 	  1
+//│ ╙──       	  ^
 //│ = [function id]
 
 :e
@@ -111,10 +151,10 @@ id(0
 //│ |id|(|0|→|id|←|)|(|1|)|
 //│ Parsed:
 //│ 	App(App(Ident(id),Tup(List(Jux(IntLit(0),Block(List(Ident(id))))))),Tup(List(IntLit(1))))
-//│ ╔══[ERROR] Illegal juxtaposition right-hand side.
-//│ ║  l.110: 	  id)(1)
+//│ ╔══[ERROR] Illegal juxtaposition right-hand side (identifier).
+//│ ║  l.150: 	  id)(1)
 //│ ╙──       	  ^^
-//│ ═══[RUNTIME ERROR] TypeError: tmp18 is not a function
+//│ ═══[RUNTIME ERROR] TypeError: tmp25 is not a function
 
 
 fun test2() =
@@ -158,16 +198,10 @@ module P with
     )
   module TraceLogger
 
-:pe
 module P with
   (
   )
   module TraceLogger
-//│ ╔══[PARSE ERROR] Expected an expression; found new line instead
-//│ ║  l.163: 	  (
-//│ ║         	   ^
-//│ ║  l.164: 	  )
-//│ ╙──       	^^
 
 
 module P with ...

hkmc2/shared/src/test/mlscript/basics/Modules.mls

@@ -17,7 +17,7 @@ module Foo with val x = 1
 //│ ╔══[PARSE ERROR] Expected end of input; found '=' keyword instead
 //│ ║  l.13: 	module Foo with val x = 1
 //│ ╙──      	                      ^
-//│ ╔══[ERROR] Illegal juxtaposition right-hand side.
+//│ ╔══[ERROR] Illegal juxtaposition right-hand side (identifier).
 //│ ║  l.13: 	module Foo with val x = 1
 //│ ╙──      	                    ^
 

hkmc2/shared/src/test/mlscript/basics/New.mls

@@ -28,13 +28,31 @@ new
   Foo()
 //│ ═══[RUNTIME ERROR] TypeError: Class constructor Foo cannot be invoked without 'new'
 
-:todo
+:e
 new
   Foo
     ()
-//│ ╔══[ERROR] Illegal juxtaposition right-hand side.
+//│ ╔══[ERROR] Illegal juxtaposition right-hand side (unit).
 //│ ║  l.34: 	    ()
 //│ ╙──      	    ^^
 //│ = Foo
 
+:w
+:re
+new
+  Foo
+  ()
+//│ ╔══[WARNING] Pure expression in statement position
+//│ ║  l.43: 	  Foo
+//│ ╙──      	  ^^^
+//│ ═══[RUNTIME ERROR] TypeError: runtime.Unit is not a constructor
+
+:e
+new Foo
+  ()
+//│ ╔══[ERROR] Illegal juxtaposition right-hand side (unit).
+//│ ║  l.52: 	  ()
+//│ ╙──      	  ^^
+//│ = Foo
+