Run tests without optimization (#851)

Closes #846 

This runs every test without optimization *in addition* to running them
with optimization. While this is good for testing, we should change this
once it's mergable so our CI doesn't take ~~twice as long~~ much longer.

There are many new errors now. I've spent some time investigating and
they generally fall into these two categories:

- no block info (unsoundness in the optimizer)
- unsupported LLVM feature (toplevel object definitions, reached hole
(no LLVM FFI for extern definition)), which were somehow optimized away
- valgrind error

Author: marvinborner

effekt/jvm/src/test/scala/effekt/EffektTests.scala

@@ -36,9 +36,12 @@ trait EffektTests extends munit.FunSuite {
 
   def negatives: List[File] = List()
 
+  // Test files that should be run with optimizations disabled
+  def withoutOptimizations: List[File] = List()
+
   def runTestFor(input: File, expected: String): Unit =
     test(input.getPath + s" (${backendName})") {
-      assertNoDiff(run(input), expected)
+      assertNoDiff(run(input, true), expected)
     }
 
   // one shared driver for all tests in this test runner
@@ -59,7 +62,7 @@ trait EffektTests extends munit.FunSuite {
     compiler.compileFile(input.getPath, configs)
     compiler.context.backup
 
-  def run(input: File): String =
+  def run(input: File, optimizations: Boolean): String =
     val compiler = driver
     var options = Seq(
       "--Koutput", "string",
@@ -68,6 +71,7 @@ trait EffektTests extends munit.FunSuite {
     )
     if (valgrind) options = options :+ "--valgrind"
     if (debug) options = options :+ "--debug"
+    if (!optimizations) options = options :+ "--no-optimize"
     val configs = compiler.createConfig(options)
     configs.verify()
 
@@ -96,22 +100,32 @@ trait EffektTests extends munit.FunSuite {
       case Right(value) =>
         negatives.foreach(runNegativeTestsIn)
         positives.foreach(runPositiveTestsIn)
+        withoutOptimizations.foreach(runWithoutOptimizations)
+    }
+
+  def runWithoutOptimizations(dir: File): Unit =
+    foreachFileIn(dir) {
+      case (f, None) => sys error s"Missing checkfile for ${f.getPath}"
+      case (f, Some(expected)) =>
+        test(s"${f.getPath} (${backendName})") {
+          assertNoDiff(run(f, false), expected)
+        }
     }
 
   def runPositiveTestsIn(dir: File): Unit =
     foreachFileIn(dir) {
       case (f, None) => sys error s"Missing checkfile for ${f.getPath}"
       case (f, Some(expected)) =>
         test(s"${f.getPath} (${backendName})") {
-          assertNoDiff(run(f), expected)
+          assertNoDiff(run(f, true), expected)
         }
     }
 
   def runNegativeTestsIn(dir: File): Unit =
     foreachFileIn(dir) {
       case (f, Some(expected)) =>
         test(s"${f.getPath} (${backendName})") {
-          assertNoDiff(run(f), expected)
+          assertNoDiff(run(f, true), expected)
         }
 
       case (f, None) =>

effekt/jvm/src/test/scala/effekt/JavaScriptTests.scala

@@ -24,6 +24,18 @@ class JavaScriptTests extends EffektTests {
     examplesDir / "neg"
   )
 
+  override lazy val withoutOptimizations: List[File] = List(
+    // contifying under reset
+    //examplesDir / "pos" / "issue842.effekt",
+    //examplesDir / "pos" / "issue861.effekt",
+
+    // syntax error (multiple declaration)
+    //examplesDir / "char" / "ascii_isalphanumeric.effekt",
+    //examplesDir / "char" / "ascii_iswhitespace.effekt",
+    //examplesDir / "pos" / "parser.effekt",
+    //examplesDir / "pos" / "probabilistic.effekt",
+  )
+
   override def ignored: List[File] = List(
     // unsafe cont
     examplesDir / "pos" / "propagators.effekt"
@@ -58,7 +70,7 @@ object TestUtils {
         val shouldGenerate = regenerateAll || f.lastModified() > checkfile.lastModified()
         if (!isIgnored && shouldGenerate) {
           println(s"Writing checkfile for ${f}")
-          val out = run(f)
+          val out = run(f, true)
 
           // Save checkfile in source folder (e.g. examples/)
           // We remove ansi colors to make check files human-readable.

effekt/jvm/src/test/scala/effekt/LLVMTests.scala

@@ -53,6 +53,32 @@ class LLVMTests extends EffektTests {
     examplesDir / "pos" / "issue733.effekt",
   )
 
+  override lazy val withoutOptimizations: List[File] = List(
+    // contifying under reset
+    //examplesDir / "pos" / "issue842.effekt",
+    //examplesDir / "pos" / "issue861.effekt",
+
+    // top-level object definition
+    //examplesDir / "pos" / "object" / "if_control_effect.effekt",
+    //examplesDir / "pos" / "lambdas" / "toplevel_objects.effekt",
+    //examplesDir / "pos" / "type_omission_op.effekt",
+    //examplesDir / "pos" / "bidirectional" / "higherorderobject.effekt",
+    //examplesDir / "pos" / "bidirectional" / "res_obj_boxed.effekt",
+    //examplesDir / "pos" / "bidirectional" / "effectfulobject.effekt",
+
+    // no block info
+    //examplesDir / "pos" / "capture" / "regions.effekt",
+    //examplesDir / "pos" / "capture" / "selfregion.effekt",
+    //examplesDir / "benchmarks" / "other" / "generator.effekt",
+
+    // hole
+    //examplesDir / "pos" / "bidirectional" / "typeparametric.effekt",
+
+    // segfault
+    //examplesDir / "benchmarks" / "are_we_fast_yet" / "permute.effekt",
+    //examplesDir / "benchmarks" / "are_we_fast_yet" / "storage.effekt",
+  )
+
   override lazy val ignored: List[File] = missingFeatures ++ noValgrind(examplesDir)
 }
 

effekt/jvm/src/test/scala/effekt/StdlibTests.scala

@@ -12,13 +12,33 @@ abstract class StdlibTests extends EffektTests {
   )
 
   override def ignored: List[File] = List()
+
+  override def withoutOptimizations: List[File] = List()
 }
 
 class StdlibJavaScriptTests extends StdlibTests {
   def backendName: String = "js"
 
+  override def withoutOptimizations: List[File] = List(
+    examplesDir / "stdlib" / "acme.effekt",
+
+    //examplesDir / "stdlib" / "json.effekt",
+    //examplesDir / "stdlib" / "exception" / "combinators.effekt",
+
+    // reference error (k is not defined)
+    //examplesDir / "stdlib" / "stream" / "fibonacci.effekt",
+    //examplesDir / "stdlib" / "list" / "flatmap.effekt",
+    //examplesDir / "stdlib" / "list" / "sortBy.effekt",
+    //examplesDir / "stdlib" / "stream" / "zip.effekt",
+    //examplesDir / "stdlib" / "stream" / "characters.effekt",
+
+    // oom
+    //examplesDir / "stdlib" / "list" / "deleteat.effekt",
+  )
+
   override def ignored: List[File] = List()
 }
+
 abstract class StdlibChezTests extends StdlibTests {
   override def ignored: List[File] = List(
     // Not implemented yet
@@ -39,6 +59,10 @@ class StdlibLLVMTests extends StdlibTests {
   override def valgrind = sys.env.get("EFFEKT_VALGRIND").nonEmpty
   override def debug = sys.env.get("EFFEKT_DEBUG").nonEmpty
 
+  override def withoutOptimizations: List[File] = List(
+    examplesDir / "stdlib" / "acme.effekt",
+  )
+
   override def ignored: List[File] = List(
     // String comparison using `<`, `<=`, `>`, `>=` is not implemented yet on LLVM
     examplesDir / "stdlib" / "string" / "compare.effekt",