Fix ||| block parsing (#433)

This PR fixes 2 edges cases that sjsonnet did not handle properly.

1. ||| blocks with \r\n
2. ||| blocks only with empty - indented - lines.

The only practical way I found to fix this is to half-rewrite the
parsing code myself to better get a handle on how fastparse works.

This closes #430

Author: stephenamar-db

sjsonnet/src/sjsonnet/Parser.scala

@@ -126,25 +126,6 @@ class Parser(
   def literalSingleString[$: P]: P[Seq[String]] =
     P((CharsWhile(_ != '\'').! | "''".!.map(_ => "'")).repX ~~ "'")
 
-  def tripleBarStringLines[$: P]: P[Seq[String]] = P(
-    tripleBarStringHead.flatMapX { case (pre, w, head) =>
-      tripleBarStringBody(w).map(pre ++ Seq(head, "\n") ++ _)
-    }
-  )
-
-  def maybeChompedTripleBarString[$: P]: P[Seq[String]] = tripleBarString.map {
-    case (true, lines) =>
-      Seq(lines.mkString.stripLineEnd)
-    case (false, lines) =>
-      lines
-  }
-
-  def tripleBarString[$: P]: P[(Boolean, Seq[String])] = P(
-    ("||-" | "||").?.!.map(_.last == '-')./ ~~ CharsWhileIn(" \t", 0) ~~
-    "\n" ~~ tripleBarStringLines ~~ "\n" ~~
-    CharsWhileIn(" \t", 0) ~~ "|||"
-  )
-
   def string[$: P]: P[String] = P(
     SingleChar.flatMapX {
       case '\"' => doubleString
@@ -160,18 +141,47 @@ class Parser(
     }
   ).map(_.mkString)
 
-  def tripleBarStringHead[$: P]: P[(Seq[String], String, String)] = P(
-    (CharsWhileIn(" \t", 0) ~~ "\n".!).repX ~~
-    CharsWhileIn(" \t", 1).! ~~
-    CharsWhile(_ != '\n').!
+  def maybeChompedTripleBarString[$: P]: P[Seq[String]] = tripleBarString.map {
+    case (true, lines) =>
+      lines.dropRight(1) ++ Seq(lines.last.stripLineEnd)
+    case (false, lines) =>
+      lines
+  }
+
+  def tripleBarJunk[$: P]: P[Unit] = CharsWhileIn(" \t", 0)
+
+  def tripleBarIndent[$: P]: P[String] = P(
+    CharsWhileIn(" \t", 1)
+      .opaque("|||-block line must either be an empty line or start with at least one whitespace")
+      .!
+  )
+
+  def tripleBarString[$: P]: P[(Boolean, Seq[String])] = P(
+    ("||-" | "||").!.map(
+      _.last == '-'
+    )./ ~~ tripleBarJunk ~~ tripleBarStringLines ~~ tripleBarJunk ~~ "|||"
   )
-  def tripleBarBlankHead[$: P]: P[String] =
-    P(CharsWhileIn(" \t", 0) ~~ &("\n").map(_ => "\n"))
 
-  def tripleBarBlank[$: P]: P[String] = P("\n" ~~ tripleBarBlankHead)
+  def tripleBarStringLines[$: P]: P[Seq[String]] = P(
+    // First, we skip an empty lines until we reach the first line with indentation.
+    // This will become the indentation for the rest of the block.
+    ("\r\n" | "\n").opaque("|||-blocks require multiple lines").!./.flatMapX { case sep =>
+      (sep.!.repX ~~ tripleBarIndent)
+        .flatMapX { case (before, indent) =>
+          tripleBarStringBody(indent, sep).map(before ++ _)
+        }
+    }
+  )
 
-  def tripleBarStringBody[$: P](w: String): P[Seq[String]] = P(
-    (tripleBarBlank | "\n" ~~ w ~~ CharsWhile(_ != '\n').!.map(_ + "\n")).repX
+  def tripleBarStringBody[$: P](indent: String, sep: String): P[Seq[String]] = P(
+    // Because we already parsed the indentation, we special case the first line and only look for the content.
+    (CharsWhile(!sep.contains(_), 0) ~~ sep).!.flatMapX { firstLine =>
+      // That's the core of the parsing. Either we have an empty line or we have indentation + content + new line separator.
+      (sep.! | (indent.! ~~ (CharsWhile(!sep.contains(_), 0) ~~ sep).!).map(_._2))
+        .opaque("|||-block line must either be an empty line or start with at least one whitespace")
+        .repX
+        .map(Seq(firstLine) ++ _)
+    }
   )
 
   def arr[$: P]: P[Expr] = P((Pos ~~ &("]")).map(Val.Arr(_, emptyLazyArray)) | arrBody)

sjsonnet/test/resources/go_test_suite/std.makeArray_recursive_evalutation_order_matters.jsonnet.golden

@@ -1,39 +1 @@
-RUNTIME ERROR: max stack frames exceeded.
--------------------------------------------------
-		
-
--------------------------------------------------
-	testdata/std.makeArray_recursive_evalutation_order_matters:1:51-57	function <anonymous>
-
-local arr = [0] + std.makeArray(2000, function(i) arr[i] + 1); arr[500]
-
--------------------------------------------------
-		
-
--------------------------------------------------
-	testdata/std.makeArray_recursive_evalutation_order_matters:1:51-57	function <anonymous>
-
-local arr = [0] + std.makeArray(2000, function(i) arr[i] + 1); arr[500]
-
--------------------------------------------------
-	... (skipped 492 frames)
--------------------------------------------------
-		
-
--------------------------------------------------
-	testdata/std.makeArray_recursive_evalutation_order_matters:1:51-57	function <anonymous>
-
-local arr = [0] + std.makeArray(2000, function(i) arr[i] + 1); arr[500]
-
--------------------------------------------------
-		
-
--------------------------------------------------
-	testdata/std.makeArray_recursive_evalutation_order_matters:1:64-72	$
-
-local arr = [0] + std.makeArray(2000, function(i) arr[i] + 1); arr[500]
-
--------------------------------------------------
-	During evaluation	
-
-
+500

sjsonnet/test/resources/test_suite/dos_line_endings.jsonnet.golden

@@ -1,3 +1,3 @@
-sjsonnet.ParseError: Expected "\n":18:12, found "\r\n    A pa"
-    at .(sjsonnet/test/resources/test_suite/dos_line_endings.jsonnet:18:12)
-
+{
+   "text": "A paragraph\r\nof text.\r\n"
+}

sjsonnet/test/resources/test_suite/error.parse.string_multi_no_newline.jsonnet.golden

@@ -1,3 +1,3 @@
-sjsonnet.ParseError: Expected "\n":17:5, found "|||\n"
+sjsonnet.ParseError: Expected |||-blocks require multiple lines:17:5, found "|||\n"
     at .(sjsonnet/test/resources/test_suite/error.parse.string_multi_no_newline.jsonnet:17:5)
 

sjsonnet/test/src-js/sjsonnet/FileTests.scala

@@ -6,7 +6,6 @@ object FileTests extends BaseFileTests {
   val skippedTests = Set(
     "stdlib.jsonnet",
     "regex.jsonnet",
-    "dos_line_endings.jsonnet",
 
     // Stack size issues with the JS runner
     "recursive_function.jsonnet",
@@ -23,10 +22,8 @@ object FileTests extends BaseFileTests {
   val goTestDataSkippedTests: Set[String] = Set(
     "bitwise_or9.jsonnet",
     "builtinChar6.jsonnet",
-    "escaped_fields.jsonnet",
     "pow6.jsonnet",
     "object_invariant_plus.jsonnet",
-    "std.makeArray_recursive_evalutation_order_matters.jsonnet",
     "tailstrict3.jsonnet",
     "stdlib_smoke_test.jsonnet",
     "builtinSha1.jsonnet",

sjsonnet/test/src-jvm-native/sjsonnet/BaseFileTests.scala

@@ -97,6 +97,9 @@ abstract class BaseFileTests extends TestSuite {
             println(res.left.getOrElse(""))
             assert(e.getMessage == goldenContent.stripLineEnd)
           }
+        case e: Throwable =>
+          println(stderr.toString)
+          throw e
       }
     }
   }

sjsonnet/test/src-jvm-native/sjsonnet/FileTests.scala

@@ -4,7 +4,6 @@ import utest.*
 
 object FileTests extends BaseFileTests {
   val testDataSkippedTests: Set[String] = Set(
-    "dos_line_endings.jsonnet",
     "regex_js.jsonnet",
     "stdlib_js.jsonnet"
   ) ++ (if (isScalaNative) {
@@ -22,10 +21,8 @@ object FileTests extends BaseFileTests {
   val goTestDataSkippedTests: Set[String] = Set(
     "bitwise_or9.jsonnet",
     "builtinChar6.jsonnet",
-    "escaped_fields.jsonnet",
     "pow6.jsonnet",
     "object_invariant_plus.jsonnet",
-    "std.makeArray_recursive_evalutation_order_matters.jsonnet",
     "tailstrict3.jsonnet"
   ) ++ (if (isScalaNative)
           Set(